From bdf0f94ee2f07e607ac9897870e91b328afdaa07 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 7 May 2026 13:22:38 +0100 Subject: [PATCH 01/63] build(bitcoin-cash-node): v29.0.0 --- images/bitcoin-cash-node/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/bitcoin-cash-node/Dockerfile b/images/bitcoin-cash-node/Dockerfile index 55008df7..b84ac54b 100644 --- a/images/bitcoin-cash-node/Dockerfile +++ b/images/bitcoin-cash-node/Dockerfile @@ -7,9 +7,9 @@ RUN set -ex \ && apt-get install -qq --no-install-recommends ca-certificates gosu wget \ && rm -rf /var/lib/apt/lists/* -ENV BITCOIN_VERSION 28.0.2 -ENV BITCOIN_URL https://download.bitcoincashnode.org/misc/builds/upgrade12_temp/linux/bitcoin-cash-node-28.0.2-x86_64-linux-gnu.tar.gz -ENV BITCOIN_SHA256 140b44fd76a4f9428354bfbec4800d58fd39fb723320e761a035f15c2dd43596 +ENV BITCOIN_VERSION 29.0.0 +ENV BITCOIN_URL https://github.com/bitcoin-cash-node/bitcoin-cash-node/releases/download/v29.0.0/bitcoin-cash-node-29.0.0-x86_64-linux-gnu.tar.gz +ENV BITCOIN_SHA256 6125d1cbecc1db476f2b6b7b91da5acde92d2311b8e738124e3db64ca84b33e1 # install bitcoin binaries RUN set -ex \ From f1d7516f71b62007fd161abab1a128adef48440e Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 7 May 2026 13:28:27 +0100 Subject: [PATCH 02/63] Fixes #72 --- .cspell.json | 2 + .gitignore | 1 + .../down.sql | 2 + .../up.sql | 105 ++++++++++ src/db.ts | 7 +- src/e2e/e2e.spec.ts | 181 ++++++++++++++++++ 6 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql diff --git a/.cspell.json b/.cspell.json index 9182b0da..779f066e 100644 --- a/.cspell.json +++ b/.cspell.json @@ -103,6 +103,8 @@ "unintuitive", "unpruned", "upserting", + "UTXO", + "UTXOs", "verack", "xprivkey", "xpubkey" diff --git a/.gitignore b/.gitignore index aa314efb..249d16d4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ package-lock.json data .env client/generated +.scratch diff --git a/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/down.sql b/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/down.sql new file mode 100644 index 00000000..6def7fc0 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/down.sql @@ -0,0 +1,2 @@ +DROP TRIGGER IF EXISTS trigger_public_node_transaction_history_insert ON node_transaction_history; +DROP FUNCTION IF EXISTS trigger_node_transaction_history_insert(); diff --git a/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql b/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql new file mode 100644 index 00000000..b837b5bc --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql @@ -0,0 +1,105 @@ +CREATE OR REPLACE FUNCTION trigger_node_transaction_history_insert() RETURNS trigger + LANGUAGE plpgsql +AS $$ +BEGIN + /* + * The recursive CTE archives the full descendant set in one pass. The + * archive INSERT below re-fires this trigger, but that re-entry should return + * immediately rather than attempting another cascade. + */ + IF current_setting('chaingraph.suppress_mempool_descendant_cascade', true) = 'on' THEN + RETURN NULL; + END IF; + + -- Confirmations and empty batches do not invalidate descendants. + IF NOT EXISTS (SELECT 1 FROM new_table WHERE replaced_at IS NOT NULL) THEN + RETURN NULL; + END IF; + + PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'on', true); + + BEGIN + /* + * If another session deletes matching descendants before this DELETE + * reaches them, this archives zero rows. Suppression still prevents empty + * self-reentry. + */ + WITH RECURSIVE descendant_transactions AS ( + -- Seed: mempool transactions spending outputs of newly replaced parents. + SELECT nt.node_internal_id, + nt.transaction_internal_id, + nt.validated_at, + replaced_parents.replaced_at + FROM new_table replaced_parents + INNER JOIN transaction parent_transaction + ON parent_transaction.internal_id = replaced_parents.transaction_internal_id + INNER JOIN output parent_output + ON parent_output.transaction_hash = parent_transaction.hash + INNER JOIN input + ON input.outpoint_transaction_hash = parent_output.transaction_hash + AND input.outpoint_index = parent_output.output_index + INNER JOIN node_transaction nt + ON nt.transaction_internal_id = input.transaction_internal_id + AND nt.node_internal_id = replaced_parents.node_internal_id + WHERE replaced_parents.replaced_at IS NOT NULL + + UNION + + -- Recursive step: mempool transactions spending outputs of descendants. + SELECT child_nt.node_internal_id, + child_nt.transaction_internal_id, + child_nt.validated_at, + parent_descendants.replaced_at + FROM descendant_transactions parent_descendants + INNER JOIN transaction parent_transaction + ON parent_transaction.internal_id = parent_descendants.transaction_internal_id + INNER JOIN output parent_output + ON parent_output.transaction_hash = parent_transaction.hash + INNER JOIN input + ON input.outpoint_transaction_hash = parent_output.transaction_hash + AND input.outpoint_index = parent_output.output_index + INNER JOIN node_transaction child_nt + ON child_nt.transaction_internal_id = input.transaction_internal_id + AND child_nt.node_internal_id = parent_descendants.node_internal_id + ), + descendants AS ( + -- If reachable through multiple replaced parents, use earliest invalidation. + SELECT node_internal_id, + transaction_internal_id, + validated_at, + MIN(replaced_at) AS replaced_at + FROM descendant_transactions + GROUP BY node_internal_id, transaction_internal_id, validated_at + ), + deleted_descendants AS ( + DELETE FROM node_transaction + USING descendants + WHERE node_transaction.node_internal_id = descendants.node_internal_id + AND node_transaction.transaction_internal_id = descendants.transaction_internal_id + RETURNING node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + node_transaction.validated_at, + descendants.replaced_at + ) + INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at + FROM deleted_descendants; + EXCEPTION WHEN OTHERS THEN + PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'off', true); + RAISE; + END; + + PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'off', true); + RETURN NULL; +END; +$$; + +CREATE TRIGGER trigger_public_node_transaction_history_insert + AFTER INSERT ON node_transaction_history + REFERENCING NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE FUNCTION trigger_node_transaction_history_insert(); +COMMENT ON TRIGGER trigger_public_node_transaction_history_insert ON node_transaction_history + IS 'Cascades mempool invalidation recursively: when a node_transaction is archived to history with replaced_at set, all same-node descendants still present in node_transaction are archived with a deterministic replaced_at timestamp.'; + +-- disabled until initial sync is complete (when mempool transactions begin to be accepted) +ALTER TABLE node_transaction_history DISABLE TRIGGER trigger_public_node_transaction_history_insert; diff --git a/src/db.ts b/src/db.ts index 8c62d188..408d31f7 100644 --- a/src/db.ts +++ b/src/db.ts @@ -564,15 +564,18 @@ DELETE FROM node_block WHERE /** * After initial sync, Chaingraph begins tracking each node's mempool. * - * To maintain consistency, a trigger which is disabled before initial sync must + * To maintain consistency, triggers which are disabled before initial sync must * be reenabled to clear any confirmed or conflicting transactions when a block * is accepted. */ export const reenableMempoolCleaning = async () => { const client = await pool.connect(); - const res = await client.query( + await client.query( `ALTER TABLE node_block ENABLE TRIGGER trigger_public_node_block_insert;` ); + const res = await client.query( + `ALTER TABLE node_transaction_history ENABLE TRIGGER trigger_public_node_transaction_history_insert;` + ); client.release(); return res.rowCount; }; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 8d62a424..d7dfc2ad 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -63,6 +63,9 @@ const dbUpMigrationPaths = [ migration('default/1616195337538_init/up.sql'), migration('default/1673124945608_tokens/up.sql'), migration('default/1676794104752_parse_bytecode_pattern/up.sql'), + migration( + 'default/1778151011521_cascade_invalidate_mempool_descendants/up.sql' + ), ]; const chaingraphInternalApiPort = '3201'; @@ -616,10 +619,188 @@ test.serial('[e2e] creates expected indexes after initial sync', async (t) => { 'transaction_hash_key', 'transaction_pkey', ]); + // cspell:ignore tgenabled tgname + const triggers = ( + await client.query<{ + tgenabled: string; + tgname: string; + }>(/* sql */ ` + SELECT tgname, tgenabled FROM pg_trigger + WHERE tgname IN ( + 'trigger_public_node_block_insert', + 'trigger_public_node_transaction_history_insert' + ) + ORDER BY tgname; + `) + ).rows; + t.deepEqual(triggers, [ + { tgenabled: 'O', tgname: 'trigger_public_node_block_insert' }, + { + tgenabled: 'O', + tgname: 'trigger_public_node_transaction_history_insert', + }, + ]); clearStdoutBuffer(); t.pass(); }); +test.serial( + '[e2e] cascades replaced mempool transaction history to same-node descendants', + async (t) => { + await client.query(/* sql */ `BEGIN;`); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('parent_a', decode(repeat('f1', 32), 'hex')), + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +) +INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, 1, 0, 100, false + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('parent_a', decode(repeat('f1', 32), 'hex')), + ('child_b', decode(repeat('f3', 32), 'hex')) +) +INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + SELECT hash, 0, 1000, '\\x51'::bytea + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH input_values (child_name, parent_name, input_index) AS ( + VALUES + ('child_b', 'parent_a', 0), + ('child_c', 'child_b', 0) +), +transaction_values (name, hash) AS ( + VALUES + ('parent_a', decode(repeat('f1', 32), 'hex')), + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id, transaction.hash + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT child.internal_id, input_values.input_index, 0, 0, parent.hash, '\\x51'::bytea + FROM input_values + JOIN named_transactions child + ON child.name = input_values.child_name + JOIN named_transactions parent + ON parent.name = input_values.parent_name; +`); + await client.query(/* sql */ ` +WITH selected_nodes AS ( + SELECT name, internal_id + FROM node + WHERE name IN ('node1', 'node2') +), +transaction_values (name, hash) AS ( + VALUES + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT selected_nodes.internal_id, named_transactions.internal_id, timestamp '2026-01-01 00:00:00' + FROM selected_nodes + CROSS JOIN named_transactions; +`); + await client.query(/* sql */ ` +WITH selected_nodes AS ( + SELECT name, internal_id + FROM node + WHERE name = 'node1' +), +transaction_values (name, hash) AS ( + VALUES + ('parent_a', decode(repeat('f1', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT selected_nodes.internal_id, + named_transactions.internal_id, + timestamp '2026-01-01 00:00:00', + timestamp '2026-01-01 00:10:00' + FROM selected_nodes + CROSS JOIN named_transactions; +`); + const remainingMempool = ( + await client.query<{ + nodeName: string; + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +) +SELECT node.name AS "nodeName", transaction_values.name AS "transactionName" + FROM node_transaction + JOIN node + ON node.internal_id = node_transaction.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + JOIN transaction_values + ON transaction_values.hash = transaction.hash + ORDER BY "nodeName", "transactionName"; +`) + ).rows; + t.deepEqual(remainingMempool, [ + { nodeName: 'node2', transactionName: 'child_b' }, + { nodeName: 'node2', transactionName: 'child_c' }, + ]); + const archivedDescendants = ( + await client.query<{ + replacedAt: string; + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +) +SELECT transaction_values.name AS "transactionName", + node_transaction_history.replaced_at::text AS "replacedAt" + FROM node_transaction_history + JOIN node + ON node.internal_id = node_transaction_history.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction_history.transaction_internal_id + JOIN transaction_values + ON transaction_values.hash = transaction.hash + WHERE node.name = 'node1' + ORDER BY "transactionName"; +`) + ).rows; + t.deepEqual(archivedDescendants, [ + { replacedAt: '2026-01-01 00:10:00', transactionName: 'child_b' }, + { replacedAt: '2026-01-01 00:10:00', transactionName: 'child_c' }, + ]); + } finally { + await client.query(/* sql */ `ROLLBACK;`); + } + } +); + test.serial( '[e2e] after initial sync is complete, requests transactions as they are announced', async (t) => { From a91e50aa7e0a3c10bd601dd5d19200ade7d8d0ff Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 7 May 2026 15:01:52 +0100 Subject: [PATCH 03/63] Backfill historic mempool orhpans to history table #72 --- .../down.sql | 2 + .../up.sql | 88 +++++++++ src/e2e/e2e.spec.ts | 183 ++++++++++++++++++ 3 files changed, 273 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/down.sql b/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/down.sql new file mode 100644 index 00000000..918fd2db --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/down.sql @@ -0,0 +1,2 @@ +-- This migration archives orphaned mempool descendants into +-- node_transaction_history. The data move is intentionally not reversible. diff --git a/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/up.sql b/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/up.sql new file mode 100644 index 00000000..d90a6273 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/up.sql @@ -0,0 +1,88 @@ +DO $$ +DECLARE + previous_suppression text := current_setting('chaingraph.suppress_mempool_descendant_cascade', true); +BEGIN + /* + * The backfill archives the full existing orphan set itself. If the + * node_transaction_history trigger is enabled while this migration runs, + * suppress its re-entry from the archive INSERT below. + */ + PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'on', true); + + BEGIN + WITH RECURSIVE orphan_transactions AS ( + -- Seed: mempool transactions whose parent was already archived as replaced. + SELECT nt.node_internal_id, + nt.transaction_internal_id, + nt.validated_at, + nth.replaced_at + FROM node_transaction nt + INNER JOIN input child_input + ON child_input.transaction_internal_id = nt.transaction_internal_id + INNER JOIN transaction parent_transaction + ON parent_transaction.hash = child_input.outpoint_transaction_hash + INNER JOIN output parent_output + ON parent_output.transaction_hash = parent_transaction.hash + AND parent_output.output_index = child_input.outpoint_index + INNER JOIN node_transaction_history nth + ON nth.transaction_internal_id = parent_transaction.internal_id + AND nth.node_internal_id = nt.node_internal_id + WHERE nth.replaced_at IS NOT NULL + + UNION + + -- Recursive step: mempool transactions spending outputs of known orphans. + SELECT child_nt.node_internal_id, + child_nt.transaction_internal_id, + child_nt.validated_at, + parent_orphans.replaced_at + FROM orphan_transactions parent_orphans + INNER JOIN transaction parent_transaction + ON parent_transaction.internal_id = parent_orphans.transaction_internal_id + INNER JOIN output parent_output + ON parent_output.transaction_hash = parent_transaction.hash + INNER JOIN input child_input + ON child_input.outpoint_transaction_hash = parent_output.transaction_hash + AND child_input.outpoint_index = parent_output.output_index + INNER JOIN node_transaction child_nt + ON child_nt.transaction_internal_id = child_input.transaction_internal_id + AND child_nt.node_internal_id = parent_orphans.node_internal_id + ), + orphans AS ( + -- If reachable through multiple replaced parents, use earliest invalidation. + SELECT node_internal_id, + transaction_internal_id, + validated_at, + MIN(replaced_at) AS replaced_at + FROM orphan_transactions + GROUP BY node_internal_id, transaction_internal_id, validated_at + ), + deleted_orphans AS ( + DELETE FROM node_transaction + USING orphans + WHERE node_transaction.node_internal_id = orphans.node_internal_id + AND node_transaction.transaction_internal_id = orphans.transaction_internal_id + RETURNING node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + node_transaction.validated_at, + orphans.replaced_at + ) + INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at + FROM deleted_orphans; + EXCEPTION WHEN OTHERS THEN + PERFORM set_config( + 'chaingraph.suppress_mempool_descendant_cascade', + COALESCE(previous_suppression, 'off'), + true + ); + RAISE; + END; + + PERFORM set_config( + 'chaingraph.suppress_mempool_descendant_cascade', + COALESCE(previous_suppression, 'off'), + true + ); +END; +$$; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index d7dfc2ad..23854f65 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -59,6 +59,9 @@ const recreateDbOnStartup = true as boolean; const dir = dirname(fileURLToPath(import.meta.url)); const migration = (path: string) => resolve(dir, '../../images/hasura/hasura-data/migrations/', path); +const backfillOrphanMempoolDescendantsMigrationPath = migration( + 'default/1778158619747_backfill_orphan_mempool_descendants/up.sql' +); const dbUpMigrationPaths = [ migration('default/1616195337538_init/up.sql'), migration('default/1673124945608_tokens/up.sql'), @@ -66,6 +69,7 @@ const dbUpMigrationPaths = [ migration( 'default/1778151011521_cascade_invalidate_mempool_descendants/up.sql' ), + backfillOrphanMempoolDescendantsMigrationPath, ]; const chaingraphInternalApiPort = '3201'; @@ -801,6 +805,185 @@ SELECT transaction_values.name AS "transactionName", } ); +test.serial( + '[e2e] backfills existing orphan mempool descendants with idempotence', + async (t) => { + await client.query(/* sql */ `BEGIN;`); + // eslint-disable-next-line functional/no-try-statement + try { + const backfillMigration = readFileSync( + backfillOrphanMempoolDescendantsMigrationPath, + 'utf8' + ); + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')), + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +) +INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, 1, 0, 100, false + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')), + ('backfill_child_b', decode(repeat('e2', 32), 'hex')) +) +INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + SELECT hash, 0, 1000, '\\x51'::bytea + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH input_values (child_name, parent_name, input_index) AS ( + VALUES + ('backfill_child_b', 'backfill_parent_a', 0), + ('backfill_child_c', 'backfill_child_b', 0) +), +transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')), + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id, transaction.hash + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT child.internal_id, input_values.input_index, 0, 0, parent.hash, '\\x51'::bytea + FROM input_values + JOIN named_transactions child + ON child.name = input_values.child_name + JOIN named_transactions parent + ON parent.name = input_values.parent_name; +`); + await client.query(/* sql */ ` +WITH selected_nodes AS ( + SELECT name, internal_id + FROM node + WHERE name = 'node1' +), +transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT selected_nodes.internal_id, + named_transactions.internal_id, + timestamp '2026-01-01 00:00:00', + timestamp '2026-01-01 00:10:00' + FROM selected_nodes + CROSS JOIN named_transactions; +`); + await client.query(/* sql */ ` +WITH selected_nodes AS ( + SELECT name, internal_id + FROM node + WHERE name = 'node1' +), +transaction_values (name, hash) AS ( + VALUES + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT selected_nodes.internal_id, named_transactions.internal_id, timestamp '2026-01-01 00:00:00' + FROM selected_nodes + CROSS JOIN named_transactions; +`); + await client.query(backfillMigration); + await client.query(backfillMigration); + + const remainingMempool = ( + await client.query<{ + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +) +SELECT transaction_values.name AS "transactionName" + FROM node_transaction + JOIN node + ON node.internal_id = node_transaction.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + JOIN transaction_values + ON transaction_values.hash = transaction.hash + WHERE node.name = 'node1' + ORDER BY "transactionName"; +`) + ).rows; + t.deepEqual(remainingMempool, []); + + const archivedTransactions = ( + await client.query<{ + historyRowCount: number; + replacedAt: string; + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')), + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +) +SELECT transaction_values.name AS "transactionName", + COUNT(*)::integer AS "historyRowCount", + MIN(node_transaction_history.replaced_at)::text AS "replacedAt" + FROM node_transaction_history + JOIN node + ON node.internal_id = node_transaction_history.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction_history.transaction_internal_id + JOIN transaction_values + ON transaction_values.hash = transaction.hash + WHERE node.name = 'node1' + GROUP BY transaction_values.name + ORDER BY "transactionName"; +`) + ).rows; + t.deepEqual(archivedTransactions, [ + { + historyRowCount: 1, + replacedAt: '2026-01-01 00:10:00', + transactionName: 'backfill_child_b', + }, + { + historyRowCount: 1, + replacedAt: '2026-01-01 00:10:00', + transactionName: 'backfill_child_c', + }, + { + historyRowCount: 1, + replacedAt: '2026-01-01 00:10:00', + transactionName: 'backfill_parent_a', + }, + ]); + } finally { + await client.query(/* sql */ `ROLLBACK;`); + } + } +); + test.serial( '[e2e] after initial sync is complete, requests transactions as they are announced', async (t) => { From 59fb42b5146c2a2e3cc45763df2c76f06c928245 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sat, 9 May 2026 22:45:23 +0100 Subject: [PATCH 04/63] test(e2e): reproduce missing block transaction ingestion bug --- src/e2e/e2e.spec.ts | 48 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 23854f65..44f69b8e 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -1453,8 +1453,54 @@ test.serial( } ); +test.serial( + '[e2e] saves block transactions if previously announced tx is seen but not yet saved', + async (t) => { + const tipStartIndex = 162; + const [, tx1] = tipA[tipStartIndex]!.transactions; + mempool[swapEndianness(tx1!.hash)] = false; + const node1RequestedTx = new Promise((res) => { + node1.once('peergetdata', (_, message) => { + res(message.inventory); + }); + }); + peers.node1.sendMessage( + peers.node1.messages.Inventory.forTransaction( + Buffer.from(tx1!.hash, 'hex') + ) + ); + await node1RequestedTx; + logger.debug( + `node1: announced tipA[${tipStartIndex}] transaction 1 without providing the transaction: ${ + tx1!.hash + }` + ); + newBlocks('node1', [tipA[tipStartIndex]!]); + newBlocks('node2', [tipB[tipStartIndex]!]); + newBlocks('node3', [tipA[tipStartIndex]!]); + await waitForStdout(/Saved new block – height:\s+3163[^\n]+nodes: node2/u); + await waitForStdout( + /Saved new block – height:\s+3163[^\n]+nodes: node1, node4/u + ); + const blockTransactionCount = ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*) FROM block_transaction + INNER JOIN block ON block.internal_id = block_transaction.block_internal_id + WHERE block.hash = $1; + `, + [hexToBin(tipA[tipStartIndex]!.header.hash)] + ) + ).rows[0]!.count; + t.deepEqual( + blockTransactionCount, + tipA[tipStartIndex]!.transactions.length.toString() + ); + } +); + test.serial('[e2e] syncs remaining blocks one-by-one', async (t) => { - const tipStartIndex = 162; + const tipStartIndex = 163; slowFeedBlocks('node1', tipA.slice(tipStartIndex), 1); slowFeedBlocks('node2', tipB.slice(tipStartIndex), 1); /** From f0e3567d876a2b1840b7d9c75341b9949ac5ad89 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sat, 9 May 2026 22:54:35 +0100 Subject: [PATCH 05/63] fix(agent): save block transactions despite cache hit if db = false --- src/db.ts | 94 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/src/db.ts b/src/db.ts index 408d31f7..3b4dd5d8 100644 --- a/src/db.ts +++ b/src/db.ts @@ -300,7 +300,7 @@ export const saveBlock = async ({ }>( (transactions, transaction) => { // eslint-disable-next-line @typescript-eslint/no-unused-expressions - transactionCache.has(transaction.hash) + transactionCache.get(transaction.hash)?.db === true ? transactions.alreadySaved.push(transaction) : transactions.unknown.push(transaction); return transactions; @@ -453,11 +453,6 @@ inserted_block (internal_id) AS ( ON CONFLICT ON CONSTRAINT "block_hash_key" DO NOTHING RETURNING internal_id ), -inserted_block_transactions AS ( - INSERT INTO block_transaction (block_internal_id, transaction_internal_id, transaction_index) - SELECT blk.internal_id, tx.internal_id, tx.transaction_index - FROM inserted_block blk CROSS JOIN joined_transactions tx -), new_or_existing_block (internal_id) AS ( SELECT COALESCE ( (SELECT internal_id FROM inserted_block), @@ -465,27 +460,78 @@ new_or_existing_block (internal_id) AS ( block.hash )}'::bytea) ) -) -INSERT INTO node_block (node_internal_id, block_internal_id, accepted_at) +), +inserted_block_transactions AS ( + INSERT INTO block_transaction (block_internal_id, transaction_internal_id, transaction_index) + SELECT blk.internal_id, tx.internal_id, tx.transaction_index + FROM new_or_existing_block blk CROSS JOIN joined_transactions tx + ON CONFLICT ON CONSTRAINT "block_transaction_pkey" DO NOTHING + RETURNING transaction_internal_id +), +inserted_node_blocks AS ( + INSERT INTO node_block (node_internal_id, block_internal_id, accepted_at) SELECT node.node_internal_id, blk.internal_id, node.accepted_at FROM new_or_existing_block blk CROSS JOIN accepting_nodes node - ON CONFLICT ON CONSTRAINT "node_block_pkey" DO NOTHING`; + ON CONFLICT ON CONSTRAINT "node_block_pkey" DO NOTHING + RETURNING block_internal_id +) +SELECT + (SELECT COUNT(*)::bigint FROM joined_transactions) AS "joinedTransactionCount", + (SELECT COUNT(*)::bigint FROM inserted_block_transactions) AS "insertedBlockTransactionCount", + (SELECT COUNT(*)::bigint FROM inserted_node_blocks) AS "insertedNodeBlockCount";`; const client = await pool.connect(); - await client.query('BEGIN;'); - const saveTransactionsResult = await client.query<{ count: string }>( - addAllTransactions - ); - const attemptedSavedTransactions = blockTransactions.unknown; - const savedTransactionCount = Number(saveTransactionsResult.rows[0]!.count); - const transactionCacheMisses = - attemptedSavedTransactions.length - savedTransactionCount; - await client.query(addBlockQuery); - await client.query('COMMIT;'); - client.release(); - return { - attemptedSavedTransactions, - transactionCacheMisses, - }; + // eslint-disable-next-line functional/no-try-statement + try { + await client.query('BEGIN;'); + const saveTransactionsResult = await client.query<{ count: string }>( + addAllTransactions + ); + const attemptedSavedTransactions = blockTransactions.unknown; + const savedTransactionCount = Number(saveTransactionsResult.rows[0]!.count); + const transactionCacheMisses = + attemptedSavedTransactions.length - savedTransactionCount; + const addBlockResult = await client.query<{ + insertedBlockTransactionCount: string; + insertedNodeBlockCount: string; + joinedTransactionCount: string; + }>(addBlockQuery); + const joinedTransactionCount = Number( + addBlockResult.rows[0]!.joinedTransactionCount + ); + const linkedBlockTransactionCount = Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*)::bigint AS count + FROM block_transaction + INNER JOIN block ON block.internal_id = block_transaction.block_internal_id + WHERE block.hash = $1; + `, + [Buffer.from(block.hash, 'hex')] + ) + ).rows[0]!.count + ); + if ( + joinedTransactionCount !== block.transactions.length || + linkedBlockTransactionCount !== block.transactions.length + ) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Failed to save all transactions for block ${block.height} (${block.hash}): joined ${joinedTransactionCount}/${block.transactions.length}, linked ${linkedBlockTransactionCount}/${block.transactions.length}.` + ); + } + await client.query('COMMIT;'); + return { + attemptedSavedTransactions, + transactionCacheMisses, + }; + } catch (err) { + await client.query('ROLLBACK;'); + // eslint-disable-next-line functional/no-throw-statement + throw err; + } finally { + client.release(); + } }; /** From f89be63f5be9c60806b8846ab369f5a6b89473a7 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 10:57:39 +0100 Subject: [PATCH 06/63] fix(agent): self-heal incomplete block transactions #74 --- charts/chaingraph/templates/agent.yaml | 2 + charts/chaingraph/values.yaml | 7 +- defaults.env | 7 + src/agent.ts | 221 +++++++++++++++++++++++++ src/config.ts | 15 ++ src/db.ts | 144 ++++++++++++++++ src/e2e/e2e.spec.ts | 91 ++++++++++ 7 files changed, 485 insertions(+), 2 deletions(-) diff --git a/charts/chaingraph/templates/agent.yaml b/charts/chaingraph/templates/agent.yaml index 5b1c17fc..2388432b 100644 --- a/charts/chaingraph/templates/agent.yaml +++ b/charts/chaingraph/templates/agent.yaml @@ -41,6 +41,8 @@ spec: value: {{ .Values.agent.synchronousCommit | quote }} - name: CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB value: {{ .Values.agent.blockBufferTargetSizeMb | quote }} + - name: CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE + value: {{ .Values.agent.repairBlocksBatchSize | quote }} - name: CHAINGRAPH_GENESIS_BLOCKS value: {{ .Values.agent.genesisBlocks | quote }} - name: CHAINGRAPH_TRUSTED_NODES diff --git a/charts/chaingraph/values.yaml b/charts/chaingraph/values.yaml index 88871270..68616cee 100644 --- a/charts/chaingraph/values.yaml +++ b/charts/chaingraph/values.yaml @@ -18,12 +18,15 @@ agent: # The maximum number of connections the agent should maintain to the database. # For best performance, this should be set to the number of CPUs available to Postgres. If not set, Chaingraph will assume that Postgres is running on hardware equivalent to its own. (This is ideal if Postgres is running on either the same machine or an equivalent one from a homogenous Kubernetes node pool.) maxConnections: '' -# If set to false, the Postgres database will be configured to use "synchronous_commit = off" during initial sync. -# In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default. + # If set to false, the Postgres database will be configured to use "synchronous_commit = off" during initial sync. + # In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default. synchronousCommit: true # The target size (in MB) of the buffer which holds downloaded blocks waiting to be saved to the database. This primarily affects memory usage during the initial chain sync. # For best performance, this should be around `maxConnections * maximum block size`, while leaving enough memory available to the host machine. If left unset (recommended), Chaingraph will measure free memory at startup and attempt to select a reasonable value. blockBufferTargetSizeMb: '' + # Self-healing for https://github.com/bitauth/chaingraph/issues/74: after initial sync, the agent scans saved blocks in batches to find blocks whose linked transactions don't match the saved block size, then repairs those blocks by re-requesting them from trusted nodes. + # Set to 0 to disable the startup repair task. + repairBlocksBatchSize: 10000 # A mapping of network magic bytes to hex-encoded genesis blocks. # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated. # E.g. CHAINGRAPH_GENESIS_BLOCKS=e3e1f3e8:rawblockhex,deadbeef:rawblockhex diff --git a/defaults.env b/defaults.env index f6f87fca..6ec898d2 100644 --- a/defaults.env +++ b/defaults.env @@ -16,6 +16,13 @@ CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT=true # For best performance, this should be around `CHAINGRAPH_POSTGRES_MAX_CONNECTIONS * maximum block size`, while leaving enough memory available to the host machine. If not set, Chaingraph will measure free memory at startup and attempt to select a reasonable value. CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB= +# Self-healing for https://github.com/bitauth/chaingraph/issues/74: after +# initial sync, the agent scans saved blocks in batches to find blocks whose +# linked transactions don't match the saved block size, then repairs those +# blocks by re-requesting them from trusted nodes. Set to 0 to disable the +# startup repair task. +CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE=10000 + # A mapping of network magic bytes to hex-encoded genesis blocks. # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated. # E.g. CHAINGRAPH_GENESIS_BLOCKS=e3e1f3e8:rawblockhex,deadbeef:rawblockhex diff --git a/src/agent.ts b/src/agent.ts index 3241571a..643c532b 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -31,6 +31,7 @@ import { chaingraphLogFirehose, chaingraphUserAgent, genesisBlocks, + incompleteBlockRepairBatchSize, postgresMaxConnections, trustedNodes, } from './config.js'; @@ -38,6 +39,7 @@ import { acceptBlocksViaHeaders, createIndexes, getAllKnownBlockHashes, + getIncompleteBlocks, getIndexCreationProgress, listExistingIndexes, optionallyDisableSynchronousCommit, @@ -50,6 +52,7 @@ import { saveBlock, saveTransactionForNodes, } from './db.js'; +import type { IncompleteBlock } from './db.js'; import type { ChaingraphBlock } from './types/chaingraph.js'; // eslint-disable-next-line @typescript-eslint/naming-convention @@ -288,6 +291,23 @@ export class Agent { scheduledBlockBufferFill = false; + scheduledIncompleteBlockRepair = false; + + incompleteBlockRepairInProgress = false; + + currentIncompleteBlockRepair: + | { + hash: string; + resolve: () => void; + } + | undefined; + + incompleteBlockRepairTimeout: ReturnType | undefined; + + incompleteBlockRepairNextHeight = 0; + + completedIncompleteBlockRepairScan = false; + /** * The next second after which to log another warning that one or more nodes * are unresponsive. @@ -828,6 +848,7 @@ export class Agent { return reenableMempoolCleaning().then(() => { this.logger.info('Agent: enabled mempool tracking.'); this.saveInboundTransactions = true; + this.scheduleIncompleteBlockRepair(); }); }) .catch((err) => { @@ -906,6 +927,198 @@ export class Agent { return indexCreationCompletion; } + canScheduleIncompleteBlockRepair() { + return ![ + incompleteBlockRepairBatchSize === 0, + !this.completedInitialSync, + this.scheduledIncompleteBlockRepair, + this.incompleteBlockRepairInProgress, + this.completedIncompleteBlockRepairScan, + this.willShutdown, + ].includes(true); + } + + scheduleIncompleteBlockRepair() { + if (!this.canScheduleIncompleteBlockRepair()) { + return; + } + this.incompleteBlockRepairTimeout = setTimeout(() => { + this.scheduledIncompleteBlockRepair = false; + this.repairIncompleteBlocks().catch((err) => { + this.logger.fatal(err); + this.shutdown().catch((shutdownErr) => { + this.logger.error(shutdownErr); + }); + }); + }); + this.scheduledIncompleteBlockRepair = true; + } + + getIncompleteBlockRepairRange() { + const bestHeight = Math.max( + ...Object.values(this.blockTree.getBestHeights()) + ); + const finalHeight = bestHeight + 1; + const heightLowerBound = + this.incompleteBlockRepairNextHeight > bestHeight + ? 0 + : this.incompleteBlockRepairNextHeight; + const heightUpperBound = Math.min( + heightLowerBound + incompleteBlockRepairBatchSize, + finalHeight + ); + return { finalHeight, heightLowerBound, heightUpperBound }; + } + + updateIncompleteBlockRepairProgress({ + finalHeight, + heightLowerBound, + heightUpperBound, + incompleteBlockCount, + limit, + }: { + finalHeight: number; + heightLowerBound: number; + heightUpperBound: number; + incompleteBlockCount: number; + limit: number; + }) { + if (incompleteBlockCount === limit) { + this.incompleteBlockRepairNextHeight = heightLowerBound; + return; + } + if (heightUpperBound === finalHeight) { + this.incompleteBlockRepairNextHeight = 0; + this.completedIncompleteBlockRepairScan = true; + this.logger.info('Agent: completed incomplete block repair scan.'); + return; + } + this.incompleteBlockRepairNextHeight = heightUpperBound; + } + + async repairIncompleteBlock(block: IncompleteBlock) { + if (this.currentIncompleteBlockRepair !== undefined) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Agent: attempted to repair incomplete block ${block.height} (${block.hash}) while already repairing ${this.currentIncompleteBlockRepair.hash}.` + ); + } + const sourceNodes = this.blockTree.getNodesWithBlock( + block.hash, + block.height + ); + if (sourceNodes.length === 0) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Agent: incomplete block ${block.height} (${block.hash}) is not currently accepted by any connected node.` + ); + } + this.logger.info( + `Agent: self-healing incomplete block ${block.height} (${block.hash}); linked size ${block.linkedSizeBytes}/${block.sizeBytes} bytes across ${block.transactionCount} saved transaction(s).` + ); + const alreadyDownloading = this.blockDownloads.some( + (download) => download.hash === block.hash + ); + return new Promise((resolve) => { + this.currentIncompleteBlockRepair = { + hash: block.hash, + resolve, + }; + if (!alreadyDownloading) { + this.blockBuffer.reserveBlock(); + this.requestBlock(block.hash, block.height); + } + }); + } + + canRepairIncompleteBlocks() { + return ( + incompleteBlockRepairBatchSize !== 0 && + this.completedInitialSync && + !this.willShutdown + ); + } + + getRegisteredNodeInternalIds() { + return Object.values(this.nodes) + .map((node) => node.internalId) + .filter((id): id is number => id !== undefined); + } + + async repairIncompleteBlocksSequentially(blocks: IncompleteBlock[]) { + await blocks.reduce>(async (previousRepair, block) => { + await previousRepair; + await this.repairIncompleteBlock(block); + }, Promise.resolve()); + } + + async repairIncompleteBlocksOnce() { + const { finalHeight, heightLowerBound, heightUpperBound } = + this.getIncompleteBlockRepairRange(); + const scanStartTime = Date.now(); + const { incompleteBlocks, scannedBlockCount } = await getIncompleteBlocks({ + excludedBlockHashes: [], + heightLowerBound, + heightUpperBound, + limit: incompleteBlockRepairBatchSize, + nodeInternalIds: this.getRegisteredNodeInternalIds(), + }); + const scanDurationMs = Date.now() - scanStartTime; + const scanRate = + scanDurationMs === 0 + ? scannedBlockCount * msPerSecond + : Math.round((scannedBlockCount / scanDurationMs) * msPerSecond); + const scanPerformanceLog = `scanned ${scannedBlockCount.toLocaleString()} block(s) in ${scanDurationMs.toLocaleString()}ms (${scanRate.toLocaleString()} blocks/s)`; + if (incompleteBlocks.length === 0) { + this.updateIncompleteBlockRepairProgress({ + finalHeight, + heightLowerBound, + heightUpperBound, + incompleteBlockCount: incompleteBlocks.length, + limit: incompleteBlockRepairBatchSize, + }); + this.logger.info( + `Agent: no incomplete blocks found from height ${heightLowerBound} to ${ + heightUpperBound - 1 + }; ${scanPerformanceLog}.` + ); + return; + } + this.logger.warn( + `Agent: found ${ + incompleteBlocks.length + } incomplete block(s) from height ${heightLowerBound} to ${ + heightUpperBound - 1 + }; ${scanPerformanceLog}; requesting full block contents for repair.` + ); + await this.repairIncompleteBlocksSequentially(incompleteBlocks); + } + + /** + * Audit a bounded range of blocks accepted by currently-connected nodes. If + * the saved transactions don't sum to the saved block size, re-request the + * full block and let the normal block-saving path repair missing rows. + */ + async repairIncompleteBlocks() { + if (!this.canRepairIncompleteBlocks()) { + return; + } + if (this.incompleteBlockRepairInProgress) { + return; + } + this.incompleteBlockRepairInProgress = true; + await this.repairIncompleteBlocksOnce() + .then(() => { + this.incompleteBlockRepairInProgress = false; + this.scheduleIncompleteBlockRepair(); + }) + .catch((err) => { + this.incompleteBlockRepairInProgress = false; + // eslint-disable-next-line functional/no-throw-statement + throw err; + }); + } + /** * Internal method used to identify sync status of all nodes. Returns a list * of nodes in order from least-synced to most-synced. (Nodes which have not @@ -1417,6 +1630,11 @@ export class Agent { blockTimestampToDate(block.timestamp) ); }); + const currentRepair = this.currentIncompleteBlockRepair; + if (currentRepair?.hash === block.hash) { + currentRepair.resolve(); + this.currentIncompleteBlockRepair = undefined; + } this.blockBuffer.removeBlock(block); } @@ -1754,6 +1972,9 @@ export class Agent { this.willShutdown = true; clearInterval(eventLoopDurationInterval); clearInterval(this.heartbeatInterval); + if (this.incompleteBlockRepairTimeout !== undefined) { + clearTimeout(this.incompleteBlockRepairTimeout); + } Object.values(this.nodes).forEach((connection) => { connection.disconnect(); }); diff --git a/src/config.ts b/src/config.ts index 46f51ff1..76435f18 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,6 +36,7 @@ const configuration = { const expectedOptions = [ 'CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB', 'CHAINGRAPH_GENESIS_BLOCKS', + 'CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE', 'CHAINGRAPH_INTERNAL_API_PORT', 'CHAINGRAPH_LOG_FIREHOSE', 'CHAINGRAPH_LOG_LEVEL_STDOUT', @@ -112,6 +113,19 @@ if (isNaN(blockBufferTargetSizeMb) || blockBufferTargetSizeMb <= 0) { ); } +const incompleteBlockRepairBatchSize = Number( + configuration.CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE +); +if ( + !Number.isInteger(incompleteBlockRepairBatchSize) || + incompleteBlockRepairBatchSize < 0 +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + 'The CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE environment variable must be an integer greater than or equal to 0.' + ); +} + const extendTildeAndResolvePath = (path: string) => path.startsWith('~') ? resolve(join(homedir(), path.slice(1))) @@ -376,6 +390,7 @@ export { chaingraphLogLevelPath, chaingraphUserAgent, genesisBlocks, + incompleteBlockRepairBatchSize, postgresMaxConnections, postgresConnectionString, postgresSynchronousCommit, diff --git a/src/db.ts b/src/db.ts index 3b4dd5d8..e9ba48a5 100644 --- a/src/db.ts +++ b/src/db.ts @@ -93,6 +93,150 @@ export const getAllKnownBlockHashes = async () => { return hashes; }; +export interface IncompleteBlock { + hash: string; + height: number; + linkedSizeBytes: number; + sizeBytes: number; + transactionCount: number; +} + +export interface IncompleteBlockScan { + incompleteBlocks: IncompleteBlock[]; + scannedBlockCount: number; +} + +/** + * Find blocks for which the locally saved block_transaction rows don't sum to + * the block's saved byte size. This avoids the SQL block encoder so it can + * detect incomplete blocks even if encoder functions have bugs (e.g. #75). + */ +export const getIncompleteBlocks = async ({ + heightLowerBound, + heightUpperBound, + limit, + nodeInternalIds, + excludedBlockHashes, +}: { + excludedBlockHashes: string[]; + heightLowerBound: number; + heightUpperBound: number; + limit: number; + nodeInternalIds: number[]; +}): Promise => { + if (nodeInternalIds.length === 0) { + return { incompleteBlocks: [], scannedBlockCount: 0 }; + } + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const incompleteBlockScan = await client.query<{ + incompleteBlocks: { + hash: string; + height: number | string; + linkedSizeBytes: number | string; + sizeBytes: number | string; + transactionCount: number | string; + }[]; + scannedBlockCount: string; + }>( + /* sql */ ` +WITH linked_transactions AS ( + SELECT + block.internal_id, + block.height, + block.hash, + block.size_bytes, + COUNT(block_transaction.transaction_internal_id)::bigint + AS transaction_count, + COALESCE(SUM(transaction.size_bytes), 0)::bigint + AS transaction_size_bytes + FROM block + LEFT JOIN block_transaction + ON block_transaction.block_internal_id = block.internal_id + LEFT JOIN transaction + ON transaction.internal_id = block_transaction.transaction_internal_id + WHERE block.height >= $2 + AND block.height < $3 + AND NOT (encode(block.hash, 'hex') = ANY($5::text[])) + AND EXISTS ( + SELECT 1 FROM node_block + WHERE node_block.block_internal_id = block.internal_id + AND node_block.node_internal_id = ANY($1::integer[]) + ) + GROUP BY block.internal_id +), +linked_block_sizes AS ( + SELECT + hash, + height, + size_bytes, + transaction_count, + 80 + + CASE + WHEN transaction_count <= 252 THEN 1 + WHEN transaction_count <= 65535 THEN 3 + WHEN transaction_count <= 4294967295 THEN 5 + ELSE 9 + END + + transaction_size_bytes AS linked_size_bytes + FROM linked_transactions +), +incomplete_blocks AS ( + SELECT + encode(hash, 'hex') AS hash, + height, + linked_size_bytes, + size_bytes, + transaction_count + FROM linked_block_sizes + WHERE linked_size_bytes != size_bytes + ORDER BY height ASC, hash ASC + LIMIT $4 +) +SELECT + (SELECT COUNT(*)::bigint FROM linked_block_sizes) AS "scannedBlockCount", + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'hash', hash, + 'height', height, + 'linkedSizeBytes', linked_size_bytes, + 'sizeBytes', size_bytes, + 'transactionCount', transaction_count + ) + ORDER BY height ASC, hash ASC + ) + FROM incomplete_blocks + ), + '[]'::jsonb + ) AS "incompleteBlocks"; +`, + [ + nodeInternalIds, + heightLowerBound, + heightUpperBound, + limit, + excludedBlockHashes, + ] + ); + const scan = incompleteBlockScan.rows[0]!; + return { + incompleteBlocks: scan.incompleteBlocks.map((block) => ({ + hash: block.hash, + height: Number(block.height), + linkedSizeBytes: Number(block.linkedSizeBytes), + sizeBytes: Number(block.sizeBytes), + transactionCount: Number(block.transactionCount), + })), + scannedBlockCount: Number(scan.scannedBlockCount), + }; + } finally { + client.release(); + } +}; + /** * Create or update one or more trusted node in the Chaingraph database, * returning it's internal ID. diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 44f69b8e..1c3a3358 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -145,6 +145,7 @@ const postgresE2eConnectionStringTestDb = `${postgresE2eConnectionStringBase}/${ const e2eEnvVariables = { /* eslint-disable @typescript-eslint/naming-convention */ CHAINGRAPH_GENESIS_BLOCKS: `${e2eTestNetworkMagicHex}:${genesisBlockRaw},e3e1f3e8:${genesisBlockRaw},dab5bffa:${testnetGenesisBlockRaw}`, + CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE: '10000', CHAINGRAPH_INTERNAL_API_PORT: chaingraphInternalApiPort, CHAINGRAPH_LOG_FIREHOSE: logP2pMessage.toString(), CHAINGRAPH_LOG_PATH: chaingraphE2eLogPath, @@ -564,6 +565,37 @@ const sleep = async (ms: number) => new Promise((res) => { setTimeout(res, ms); }); +const blockRepairPollingAttempts = 40; +const blockRepairPollingIntervalMs = 250; +const getBlockTransactionCount = async (blockHash: string) => + Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*) FROM block_transaction + INNER JOIN block ON block.internal_id = block_transaction.block_internal_id + WHERE block.hash = $1; + `, + [hexToBin(blockHash)] + ) + ).rows[0]!.count + ); +const waitForBlockTransactionCount = async ( + blockHash: string, + expectedCount: number, + remainingAttempts = blockRepairPollingAttempts +): Promise => { + const count = await getBlockTransactionCount(blockHash); + if (count === expectedCount || remainingAttempts === 0) { + return count; + } + await sleep(blockRepairPollingIntervalMs); + return waitForBlockTransactionCount( + blockHash, + expectedCount, + remainingAttempts - 1 + ); +}; test.serial( '[e2e] ignores inbound transactions before initial sync is complete', @@ -1393,6 +1425,51 @@ test.serial('[e2e] shuts down with SIGINT', async (t) => { t.pass(); }); +const historicalRepairTipIndex = 161; +const historicalRepairTransactionIndex = 1; +const historicalRepairBlock = tipA[historicalRepairTipIndex]!; +const historicalRepairBlockHash = historicalRepairBlock.header.hash; + +test.serial( + '[e2e] prepares incomplete historical block transaction before restart', + async (t) => { + const transactionHash = + historicalRepairBlock.transactions[historicalRepairTransactionIndex]! + .hash; + const selectedTransaction = ( + await client.query<{ hash: string }>( + /* sql */ ` + SELECT encode(transaction.hash, 'hex') AS hash + FROM block_transaction + INNER JOIN block + ON block.internal_id = block_transaction.block_internal_id + INNER JOIN transaction + ON transaction.internal_id = + block_transaction.transaction_internal_id + WHERE block.hash = $1 + AND block_transaction.transaction_index = $2; + `, + [hexToBin(historicalRepairBlockHash), historicalRepairTransactionIndex] + ) + ).rows; + t.deepEqual(selectedTransaction, [{ hash: transactionHash }]); + await client.query( + /* sql */ ` + DELETE FROM block_transaction + USING block + WHERE block.internal_id = block_transaction.block_internal_id + AND block.hash = $1 + AND block_transaction.transaction_index = $2; + `, + [hexToBin(historicalRepairBlockHash), historicalRepairTransactionIndex] + ); + t.deepEqual( + await getBlockTransactionCount(historicalRepairBlockHash), + historicalRepairBlock.transactions.length - 1 + ); + } +); + test.serial( '[e2e] restores sync-state from database on restart (after initial sync)', async (t) => { @@ -1442,6 +1519,20 @@ test.serial('[e2e] catches up a new node via headers', async (t) => { t.pass(); }); +test.serial( + '[e2e] self-heals incomplete historical block transactions on startup', + async (t) => { + t.timeout(oneMinute); + t.deepEqual( + await waitForBlockTransactionCount( + historicalRepairBlockHash, + historicalRepairBlock.transactions.length + ), + historicalRepairBlock.transactions.length + ); + } +); + test.serial( '[e2e] handles empty headers messages (fully-synced)', async (t) => { From e0fa7f0b4b57557bf69106affe0899b745c86fe6 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 10:58:10 +0100 Subject: [PATCH 07/63] test(e2e): reproduce block.encoded_hex bug (#75 item 1) --- src/e2e/e2e.spec.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 1c3a3358..172c5a71 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -1129,6 +1129,23 @@ test.serial( } ); +test.serial( + '[e2e] get hex-encoded block with multiple transactions', + async (t) => { + const blockWithMultipleTransactions = mockchainBeforeFork[1]!; + t.true(blockWithMultipleTransactions.transactions.length > 1); + /* eslint-disable @typescript-eslint/naming-convention */ + const encodedHex = ( + await client.query<{ block_encoded_hex: string }>( + /* sql */ `SELECT block_encoded_hex(block) FROM block WHERE hash = $1::bytea;`, + [hexToBin(blockWithMultipleTransactions.header.hash)] + ) + ).rows[0]!.block_encoded_hex; + /* eslint-enable @typescript-eslint/naming-convention */ + t.deepEqual(encodedHex, binToHex(blockWithMultipleTransactions.toBuffer())); + } +); + const newBlocks = ( node: 'node1' | 'node2' | 'node3', blocks: BitcoreBlock[] From abb6d4b9d48ac686a0a6126dc6baff334e151593 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 11:11:59 +0100 Subject: [PATCH 08/63] fix(sql): encode block transaction count correctly (#75 item 1) --- .../down.sql | 25 ++++++++++++++++ .../up.sql | 30 +++++++++++++++++++ src/e2e/e2e.spec.ts | 1 + 3 files changed, 56 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/down.sql b/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/down.sql new file mode 100644 index 00000000..6eb242d3 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/down.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE FUNCTION encode_block(block_row block) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + transactions CURSOR FOR SELECT transaction.* FROM transaction + INNER JOIN block_transaction ON transaction.internal_id = block_transaction.transaction_internal_id + WHERE block_transaction.block_internal_id = block_row.internal_id + ORDER BY block_transaction.transaction_index ASC; + encoded_block bytea := encode_block_header(block_row) || encode_compact_uint(COUNT(transactions)); +BEGIN + FOR transaction_row IN transactions + LOOP + encoded_block := encoded_block || + encode_transaction(ROW( + transaction_row.internal_id, + transaction_row.hash, + transaction_row.version, + transaction_row.locktime, + transaction_row.size_bytes, + transaction_row.is_coinbase)::transaction + ); + END LOOP; + RETURN encoded_block; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/up.sql b/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/up.sql new file mode 100644 index 00000000..4e572eee --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/up.sql @@ -0,0 +1,30 @@ +CREATE OR REPLACE FUNCTION encode_block(block_row block) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + transactions CURSOR FOR SELECT transaction.* FROM transaction + INNER JOIN block_transaction ON transaction.internal_id = block_transaction.transaction_internal_id + WHERE block_transaction.block_internal_id = block_row.internal_id + ORDER BY block_transaction.transaction_index ASC; + encoded_block bytea := encode_block_header(block_row); + transaction_count bigint := 0; +BEGIN + SELECT COUNT(*) INTO transaction_count + FROM block_transaction + WHERE block_transaction.block_internal_id = block_row.internal_id; + encoded_block := encoded_block || encode_compact_uint(transaction_count); + FOR transaction_row IN transactions + LOOP + encoded_block := encoded_block || + encode_transaction(ROW( + transaction_row.internal_id, + transaction_row.hash, + transaction_row.version, + transaction_row.locktime, + transaction_row.size_bytes, + transaction_row.is_coinbase)::transaction + ); + END LOOP; + RETURN encoded_block; +END; +$$; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 172c5a71..6aa54b53 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -70,6 +70,7 @@ const dbUpMigrationPaths = [ 'default/1778151011521_cascade_invalidate_mempool_descendants/up.sql' ), backfillOrphanMempoolDescendantsMigrationPath, + migration('default/1778351597200_fix_block_encoded_transaction_count/up.sql'), ]; const chaingraphInternalApiPort = '3201'; From b6c1b4a7ae8ed7d31b953730b5c0d3d73941c9fa Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 11:25:27 +0100 Subject: [PATCH 09/63] test(e2e): reproduce invalid SQL bug in recordNodeValidation (#75 item 2) --- src/e2e/e2e.spec.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 6aa54b53..b3f01054 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -1051,6 +1051,34 @@ test.serial( } ); +test.serial( + '[e2e] records validation when another node announces a known transaction', + async (t) => { + peers.node2.sendMessage( + new peers.node2.messages.Transaction(new Transaction(halTxRaw)) + ); + const delay = 1000; + await sleep(delay); + const validations = await client.query<{ name: string }>( + /* sql */ ` + SELECT node.name + FROM node_transaction + INNER JOIN node + ON node.internal_id = node_transaction.node_internal_id + INNER JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + WHERE transaction.hash = $1 + ORDER BY node.name ASC; + `, + [hexToBin(halTxHash)] + ); + t.deepEqual( + validations.rows.map(({ name }) => name), + ['node1', 'node2'] + ); + } +); + test.serial('[e2e] handles first chipnet CashTokens transaction', async (t) => { peers.node1.sendMessage( new peers.node1.messages.Transaction(new Transaction(chipnetCashTokensTx)) From 7c4a30e3e30409ea705789176da76f6b60b488fe Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 11:58:26 +0100 Subject: [PATCH 10/63] fix(agent): invalid SQL bug in recordNodeValidation (#75 item 2) --- src/db.ts | 31 ++++++++++++++++++++----------- src/e2e/e2e.spec.ts | 11 +++++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/db.ts b/src/db.ts index e9ba48a5..b9b55576 100644 --- a/src/db.ts +++ b/src/db.ts @@ -383,8 +383,6 @@ INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validat /** * Immediately mark a node as having validated a transaction already known to * exist in the database. - * - * TODO: test */ export const recordNodeValidation = async ( transactionHash: string, @@ -397,16 +395,27 @@ export const recordNodeValidation = async ( /* * The transaction is already saved, just insert `node_transaction`s. */ - await client.query(/* sql */ ` - INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) - SELECT node_internal_id, validated_at FROM (VALUES ( + // eslint-disable-next-line functional/no-try-statement + try { + await client.query(/* sql */ ` + WITH node_transaction_values (node_internal_id, validated_at) AS ( + VALUES ( ${validation.nodeInternalId}::bigint, - ${dateToTimestampWithoutTimezone(validation.validatedAt)}) - INNER JOIN (SELECT internal_id as transaction_internal_id from transaction WHERE hash = '${hexToByteaString( - transactionHash - )}'::bytea); - `); - client.release(); + ${dateToTimestampWithoutTimezone(validation.validatedAt)} + ) + ), known_transaction (transaction_internal_id) AS ( + SELECT internal_id + FROM transaction + WHERE hash = '${hexToByteaString(transactionHash)}'::bytea + ) + INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT node_internal_id, transaction_internal_id, validated_at + FROM node_transaction_values + CROSS JOIN known_transaction; + `); + } finally { + client.release(); + } }; /** diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index b3f01054..87190196 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -1076,6 +1076,17 @@ test.serial( validations.rows.map(({ name }) => name), ['node1', 'node2'] ); + await client.query( + /* sql */ ` + DELETE FROM node_transaction + USING node, transaction + WHERE node_transaction.node_internal_id = node.internal_id + AND node_transaction.transaction_internal_id = transaction.internal_id + AND node.name = 'node2' + AND transaction.hash = $1; + `, + [hexToBin(halTxHash)] + ); } ); From b94062657163d7c62cba392f0e460159225f79c7 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 12:50:22 +0100 Subject: [PATCH 11/63] fix(agent): make node_transaction inserts idempotent --- src/db.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/db.ts b/src/db.ts index b9b55576..afe65fc4 100644 --- a/src/db.ts +++ b/src/db.ts @@ -373,7 +373,8 @@ WITH transaction_values (hash, version, locktime, size_bytes, is_coinbase) AS ( ) ) INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) - SELECT node_internal_id, transaction_internal_id, validated_at FROM node_transaction_values CROSS JOIN new_or_existing_transaction; + SELECT node_internal_id, transaction_internal_id, validated_at FROM node_transaction_values CROSS JOIN new_or_existing_transaction + ON CONFLICT ON CONSTRAINT "node_transaction_pkey" DO NOTHING; `; const client = await pool.connect(); await client.query(saveTransaction); @@ -411,7 +412,8 @@ export const recordNodeValidation = async ( INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) SELECT node_internal_id, transaction_internal_id, validated_at FROM node_transaction_values - CROSS JOIN known_transaction; + CROSS JOIN known_transaction + ON CONFLICT ON CONSTRAINT "node_transaction_pkey" DO NOTHING; `); } finally { client.release(); From 236253eb198dc23fd7dd491d263f19b951e6d148 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 13:12:07 +0100 Subject: [PATCH 12/63] test(e2e): reproduce data_carrier_outputs empty bytecode bug (#75 item 3) --- src/e2e/e2e.spec.ts | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 87190196..d9d953ae 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -1186,6 +1186,38 @@ test.serial( } ); +test.serial( + '[e2e] [postgres] transaction_data_carrier_outputs ignores empty locking bytecode', + async (t) => { + const txHash = + '0000000000000000000000000000000000000000000000000000000000000075'; + await client.query( + /* sql */ ` + INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + VALUES ($1::bytea, 1, 0, 10, false); + `, + [hexToBin(txHash)] + ); + await client.query( + /* sql */ ` + INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + VALUES ($1::bytea, 0, 1, $2::bytea); + `, + [hexToBin(txHash), hexToBin('')] + ); + const outputs = await client.query<{ outputIndex: string }>( + /* sql */ ` + SELECT output_index AS "outputIndex" + FROM transaction_data_carrier_outputs( + (SELECT transaction FROM transaction WHERE hash = $1::bytea) + ); + `, + [hexToBin(txHash)] + ); + t.deepEqual(outputs.rows, []); + } +); + const newBlocks = ( node: 'node1' | 'node2' | 'node3', blocks: BitcoreBlock[] From bbdb8fb69b0fc46ec1e49a79e5329b8d19ca914a Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 16:29:16 +0100 Subject: [PATCH 13/63] fix(sql): handle empty bytecode in data_carrier_outputs (#75 item 3) --- .../down.sql | 5 +++++ .../up.sql | 5 +++++ src/e2e/e2e.spec.ts | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/down.sql b/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/down.sql new file mode 100644 index 00000000..7608b37e --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/down.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE FUNCTION transaction_data_carrier_outputs(transaction_row transaction) RETURNS SETOF output + LANGUAGE sql IMMUTABLE +AS $$ + SELECT * FROM output WHERE transaction_hash = $1.hash AND (value_satoshis = 0 OR get_byte(locking_bytecode, 0) = 106); +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql b/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql new file mode 100644 index 00000000..b1cdd137 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE FUNCTION transaction_data_carrier_outputs(transaction_row transaction) RETURNS SETOF output + LANGUAGE sql IMMUTABLE +AS $$ + SELECT * FROM output WHERE transaction_hash = $1.hash AND (value_satoshis = 0 OR (octet_length(locking_bytecode) > 0 AND get_byte(locking_bytecode, 0) = 106)); +$$; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index d9d953ae..821451a4 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -71,6 +71,9 @@ const dbUpMigrationPaths = [ ), backfillOrphanMempoolDescendantsMigrationPath, migration('default/1778351597200_fix_block_encoded_transaction_count/up.sql'), + migration( + 'default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql' + ), ]; const chaingraphInternalApiPort = '3201'; From 767d3a4ec8b86408d23e3408add4f32343485565 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 17:04:12 +0100 Subject: [PATCH 14/63] test(e2e): reproduce coinbase-only value aggregate bug (#75 item 4) --- src/e2e/e2e.spec.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 821451a4..903582ab 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -1172,6 +1172,33 @@ test.serial( } ); +test.serial( + '[e2e] [postgres] value aggregates handle coinbase-only blocks', + async (t) => { + const aggregates = ( + await client.query<{ + feeSatoshis: string; + generatedValueSatoshis: string; + inputValueSatoshis: string; + outputValueSatoshis: string; + }>(/* sql */ ` + SELECT + block_fee_satoshis(block)::text AS "feeSatoshis", + block_generated_value_satoshis(block)::text AS "generatedValueSatoshis", + block_input_value_satoshis(block)::text AS "inputValueSatoshis", + block_output_value_satoshis(block)::text AS "outputValueSatoshis" + FROM block WHERE height = 0; + `) + ).rows[0]!; + t.deepEqual(aggregates, { + feeSatoshis: '0', + generatedValueSatoshis: '5000000000', + inputValueSatoshis: '0', + outputValueSatoshis: '5000000000', + }); + } +); + test.serial( '[e2e] get hex-encoded block with multiple transactions', async (t) => { From e6527e0aacfe5bc46493d9613db38a88413f1a77 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 17:19:41 +0100 Subject: [PATCH 15/63] fix(sql): handle coinbase-only value aggregates (#75 item 4) --- .../down.sql | 59 ++++++++++++++++++ .../up.sql | 62 +++++++++++++++++++ src/e2e/e2e.spec.ts | 1 + 3 files changed, 122 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/down.sql b/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/down.sql new file mode 100644 index 00000000..1562de57 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/down.sql @@ -0,0 +1,59 @@ +CREATE OR REPLACE FUNCTION transaction_input_value_satoshis(transaction_row transaction) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(inputs.input_value_satoshis)::bigint FROM ( + SELECT input_value_satoshis (input) FROM input WHERE transaction_internal_id = transaction_row.internal_id + ) as "inputs" +$$; + +CREATE OR REPLACE FUNCTION block_input_count(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_input_count)::bigint FROM ( + SELECT transaction_input_count (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_input_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_input_value_satoshis)::bigint FROM ( + SELECT transaction_input_value_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_output_count(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_output_count)::bigint FROM ( + SELECT transaction_output_count (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_output_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_output_value_satoshis)::bigint FROM ( + SELECT transaction_output_value_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION transaction_fee_satoshis(transaction_row transaction) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT transaction_input_value_satoshis(transaction_row) - transaction_output_value_satoshis(transaction_row) +$$; + +CREATE OR REPLACE FUNCTION block_fee_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_fee_satoshis)::bigint FROM ( + SELECT transaction_fee_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id ) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_generated_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT (block_output_value_satoshis(block) - block_input_value_satoshis(block))::bigint FROM block WHERE internal_id = block_row.internal_id; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/up.sql b/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/up.sql new file mode 100644 index 00000000..b6045378 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/up.sql @@ -0,0 +1,62 @@ +CREATE OR REPLACE FUNCTION transaction_input_value_satoshis(transaction_row transaction) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(inputs.input_value_satoshis), 0)::bigint FROM ( + SELECT input_value_satoshis (input) FROM input WHERE transaction_internal_id = transaction_row.internal_id + ) as "inputs" +$$; + +CREATE OR REPLACE FUNCTION block_input_count(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_input_count), 0)::bigint FROM ( + SELECT transaction_input_count (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_input_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_input_value_satoshis), 0)::bigint FROM ( + SELECT transaction_input_value_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_output_count(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_output_count), 0)::bigint FROM ( + SELECT transaction_output_count (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_output_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_output_value_satoshis), 0)::bigint FROM ( + SELECT transaction_output_value_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION transaction_fee_satoshis(transaction_row transaction) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT CASE + WHEN transaction_row.is_coinbase THEN 0 + ELSE transaction_input_value_satoshis(transaction_row) - transaction_output_value_satoshis(transaction_row) + END +$$; + +CREATE OR REPLACE FUNCTION block_fee_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_fee_satoshis), 0)::bigint FROM ( + SELECT transaction_fee_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id ) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_generated_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT (block_output_value_satoshis(block) - block_input_value_satoshis(block))::bigint FROM block WHERE internal_id = block_row.internal_id; +$$; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 903582ab..3d405aa9 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -74,6 +74,7 @@ const dbUpMigrationPaths = [ migration( 'default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql' ), + migration('default/1778429124205_fix_coinbase_only_value_aggregates/up.sql'), ]; const chaingraphInternalApiPort = '3201'; From 4a6f4b37c5872d7344f4220ada8a906ed886bbc8 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 18:47:24 +0100 Subject: [PATCH 16/63] fix(agent): make backwards compatible if db schema lagging (#74) --- src/agent.ts | 7 ++++++- src/db.ts | 20 ++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index 643c532b..61ca75fb 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -845,7 +845,12 @@ export class Agent { this.logger.info( `Agent: all managed indexes have been created.` ); - return reenableMempoolCleaning().then(() => { + return reenableMempoolCleaning().then((schemaIsCurrent) => { + if (!schemaIsCurrent) { + this.logger.warn( + 'Agent: WARNING! Database schema is old and missing multiple bug fixes. Update the Hasura image to apply migrations and then restart this agent.' + ); + } this.logger.info('Agent: enabled mempool tracking.'); this.saveInboundTransactions = true; this.scheduleIncompleteBlockRepair(); diff --git a/src/db.ts b/src/db.ts index afe65fc4..13a77f0e 100644 --- a/src/db.ts +++ b/src/db.ts @@ -774,11 +774,23 @@ export const reenableMempoolCleaning = async () => { await client.query( `ALTER TABLE node_block ENABLE TRIGGER trigger_public_node_block_insert;` ); - const res = await client.query( - `ALTER TABLE node_transaction_history ENABLE TRIGGER trigger_public_node_transaction_history_insert;` - ); + const triggerExists = + ( + await client.query<{ triggerExists: boolean }>(/* sql */ ` +SELECT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgrelid = 'node_transaction_history'::regclass + AND tgname = 'trigger_public_node_transaction_history_insert' +) AS "triggerExists"; +`) + ).rows[0]?.triggerExists === true; + if (triggerExists) { + await client.query( + `ALTER TABLE node_transaction_history ENABLE TRIGGER trigger_public_node_transaction_history_insert;` + ); + } client.release(); - return res.rowCount; + return triggerExists; }; /** From d9bff00185e4742fed52fd360a768ae3530b0349 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 19:03:57 +0100 Subject: [PATCH 17/63] fix(sql): add node_transaction_history primary key (#75 item 5) --- .../down.sql | 2 ++ .../up.sql | 2 ++ src/e2e/e2e.spec.ts | 4 ++++ 3 files changed, 8 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/down.sql b/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/down.sql new file mode 100644 index 00000000..6759492c --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/down.sql @@ -0,0 +1,2 @@ +ALTER TABLE ONLY node_transaction_history + DROP CONSTRAINT node_transaction_history_pkey; diff --git a/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/up.sql b/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/up.sql new file mode 100644 index 00000000..84283256 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/up.sql @@ -0,0 +1,2 @@ +ALTER TABLE ONLY node_transaction_history + ADD CONSTRAINT node_transaction_history_pkey PRIMARY KEY (internal_id); diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 3d405aa9..43fca8fa 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -75,6 +75,9 @@ const dbUpMigrationPaths = [ 'default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql' ), migration('default/1778429124205_fix_coinbase_only_value_aggregates/up.sql'), + migration( + 'default/1778435997270_add_node_transaction_history_primary_key/up.sql' + ), ]; const chaingraphInternalApiPort = '3201'; @@ -653,6 +656,7 @@ test.serial('[e2e] creates expected indexes after initial sync', async (t) => { 'node_internal_id_key', 'node_name_key', 'node_pkey', + 'node_transaction_history_pkey', 'node_transaction_pkey', 'output_pkey', 'output_search_index', From be5e686204cf8e7cc042cc705305bfeeb1b9fbae Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 19:26:10 +0100 Subject: [PATCH 18/63] test(e2e): reproduce zero-length PUSHDATA pattern bug (#75 item 6) --- src/e2e/e2e.spec.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 43fca8fa..1b9fe9fb 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -2095,6 +2095,28 @@ test( '6a02094c5c' ); +test( + 'zero-length OP_PUSHDATA1', + bytecodeFunction, + 'parse_bytecode_pattern_with_pushdata_lengths', + '4c00', + '4c00' +); +test( + 'zero-length OP_PUSHDATA2', + bytecodeFunction, + 'parse_bytecode_pattern_with_pushdata_lengths', + '4d0000', + '4d0000' +); +test( + 'zero-length OP_PUSHDATA4', + bytecodeFunction, + 'parse_bytecode_pattern_with_pushdata_lengths', + '4e00000000', + '4e00000000' +); + test( 'OP_RETURN with OP_PUSHDATA2', bytecodeFunction, From 54c07162396b0a4c499676775fe0ae0955013a44 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 19:30:45 +0100 Subject: [PATCH 19/63] fix(sql): preserve zero-length PUSHDATA length bytes (#75 item 6) --- .../down.sql | 52 +++++++++++++++++++ .../up.sql | 52 +++++++++++++++++++ src/e2e/e2e.spec.ts | 1 + 3 files changed, 105 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/down.sql b/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/down.sql new file mode 100644 index 00000000..6e599caa --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/down.sql @@ -0,0 +1,52 @@ +CREATE OR REPLACE FUNCTION parse_bytecode_pattern_with_pushdata_lengths(bytecode bytea) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + pattern bytea := '\x'::bytea; + selected_byte integer; + scratch bytea; + i integer := 0; + bytecode_length integer := octet_length(bytecode); +BEGIN + WHILE i < bytecode_length LOOP + selected_byte := get_byte(bytecode, i); + pattern := pattern || substring(bytecode from (i + 1) for 1); + IF selected_byte > 78 OR selected_byte = 0 THEN + -- OP_0 (0) and all opcodes after OP_PUSHDATA_4 (78) are single-byte instructions + i := i + 1; + ELSIF selected_byte > 0 AND selected_byte <= 75 THEN + -- OP_PUSHBYTES_1 (1) through OP_PUSHBYTES_75 (75) directly indicate the length of pushed data + i := i + 1 + selected_byte; + ELSIF selected_byte = 76 THEN + IF bytecode_length - i < 3 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_1 reads one length-byte + pattern := pattern || substring(bytecode from (i + 2) for 1); -- append length byte + i := i + 2 + get_byte(bytecode, (i + 1)); + ELSIF selected_byte = 77 THEN + IF bytecode_length - i < 4 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_2 reads two length-bytes + scratch := substring(bytecode from (i + 2) for 2); + pattern := pattern || scratch; -- append length bytes + -- parse scratch as unsigned, two byte, little-endian number: + i := i + 3 + ((get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + ELSIF selected_byte = 78 THEN + IF bytecode_length - i < 6 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_4 reads four length-bytes + scratch := substring(bytecode from (i + 2) for 4); + pattern := pattern || scratch; -- append length bytes + -- parse scratch as unsigned, four byte, little-endian number: + i := i + 5 + ((get_byte(scratch, 3) << 24) | (get_byte(scratch, 2) << 16) | (get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + END IF; + END LOOP; + RETURN pattern; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/up.sql b/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/up.sql new file mode 100644 index 00000000..34b48764 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/up.sql @@ -0,0 +1,52 @@ +CREATE OR REPLACE FUNCTION parse_bytecode_pattern_with_pushdata_lengths(bytecode bytea) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + pattern bytea := '\x'::bytea; + selected_byte integer; + scratch bytea; + i integer := 0; + bytecode_length integer := octet_length(bytecode); +BEGIN + WHILE i < bytecode_length LOOP + selected_byte := get_byte(bytecode, i); + pattern := pattern || substring(bytecode from (i + 1) for 1); + IF selected_byte > 78 OR selected_byte = 0 THEN + -- OP_0 (0) and all opcodes after OP_PUSHDATA_4 (78) are single-byte instructions + i := i + 1; + ELSIF selected_byte > 0 AND selected_byte <= 75 THEN + -- OP_PUSHBYTES_1 (1) through OP_PUSHBYTES_75 (75) directly indicate the length of pushed data + i := i + 1 + selected_byte; + ELSIF selected_byte = 76 THEN + IF bytecode_length - i < 2 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_1 reads one length-byte + pattern := pattern || substring(bytecode from (i + 2) for 1); -- append length byte + i := i + 2 + get_byte(bytecode, (i + 1)); + ELSIF selected_byte = 77 THEN + IF bytecode_length - i < 3 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_2 reads two length-bytes + scratch := substring(bytecode from (i + 2) for 2); + pattern := pattern || scratch; -- append length bytes + -- parse scratch as unsigned, two byte, little-endian number: + i := i + 3 + ((get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + ELSIF selected_byte = 78 THEN + IF bytecode_length - i < 5 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_4 reads four length-bytes + scratch := substring(bytecode from (i + 2) for 4); + pattern := pattern || scratch; -- append length bytes + -- parse scratch as unsigned, four byte, little-endian number: + i := i + 5 + ((get_byte(scratch, 3) << 24) | (get_byte(scratch, 2) << 16) | (get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + END IF; + END LOOP; + RETURN pattern; +END; +$$; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 1b9fe9fb..072538cd 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -78,6 +78,7 @@ const dbUpMigrationPaths = [ migration( 'default/1778435997270_add_node_transaction_history_primary_key/up.sql' ), + migration('default/1778437612917_fix_zero_length_pushdata_patterns/up.sql'), ]; const chaingraphInternalApiPort = '3201'; From c4218a34644cc93e14cb0fe8e10d9056e44b6071 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 19:32:43 +0100 Subject: [PATCH 20/63] fix: spelling --- src/db.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/db.ts b/src/db.ts index 13a77f0e..34ba88c3 100644 --- a/src/db.ts +++ b/src/db.ts @@ -775,6 +775,7 @@ export const reenableMempoolCleaning = async () => { `ALTER TABLE node_block ENABLE TRIGGER trigger_public_node_block_insert;` ); const triggerExists = + // cspell:ignore tgrelid tgname ( await client.query<{ triggerExists: boolean }>(/* sql */ ` SELECT EXISTS ( From 4928b9ad1fe82dace6df4459eebf13c3db846711 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 19:37:56 +0100 Subject: [PATCH 21/63] test(e2e): reproduce redeem bytecode parser bugs (#75 item 7) --- src/e2e/e2e.spec.ts | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 072538cd..102d0dcd 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -1986,6 +1986,18 @@ const bytecodeFunction = test.macro<[string, string, string]>({ `[e2e] [postgres] ${functionName} – ${patternHex}: ${providedTitle ?? ''}`, }); +const bytecodeFunctionReturnsNull = test.macro<[string, string]>({ + exec: async (t, functionName, bytecodeHex) => { + const result = await client.query<{ isNull: boolean }>( + /* sql */ `SELECT ${functionName} ($1) IS NULL AS "isNull";`, + [hexToBin(bytecodeHex)] + ); + t.true(result.rows[0]!.isNull); + }, + title: (providedTitle, functionName, bytecodeHex) => + `[e2e] [postgres] ${functionName} – ${bytecodeHex}: ${providedTitle ?? ''}`, +}); + test( 'P2PKH', bytecodeFunction, @@ -2169,10 +2181,9 @@ test( test( 'no redeem', - bytecodeFunction, + bytecodeFunctionReturnsNull, 'parse_bytecode_pattern_redeem', - `0002000051`, - '' + `0002000051` ); test( @@ -2232,6 +2243,31 @@ test( '004e515253' ); +test( + 'malformed OP_PUSHDATA1 redeem', + bytecodeFunctionReturnsNull, + 'parse_bytecode_pattern_redeem', + '4c' +); +test( + 'malformed OP_PUSHDATA2 redeem', + bytecodeFunctionReturnsNull, + 'parse_bytecode_pattern_redeem', + '4d11' +); +test( + 'malformed OP_PUSHDATA4 redeem', + bytecodeFunctionReturnsNull, + 'parse_bytecode_pattern_redeem', + '4e112233' +); +test( + 'oversized OP_PUSHDATA4 redeem', + bytecodeFunctionReturnsNull, + 'parse_bytecode_pattern_redeem', + '4effffffff' +); + test('[e2e] [postgres] encode_uint16le', async (t) => { const query = async (encoded: number) => ( From cf64c6757e3d5a93a22e4aaf9e8bd709b50619fb Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 20:05:09 +0100 Subject: [PATCH 22/63] fix(sql): harden redeem bytecode parser (fixes #75 item 7, fixes #55) --- .../down.sql | 49 +++++++++++ .../up.sql | 81 +++++++++++++++++++ src/e2e/e2e.spec.ts | 1 + 3 files changed, 131 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/down.sql b/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/down.sql new file mode 100644 index 00000000..beea7548 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/down.sql @@ -0,0 +1,49 @@ +CREATE OR REPLACE FUNCTION parse_bytecode_pattern_redeem(bytecode bytea) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + maybe_redeem bytea; + selected_byte integer; + length_value integer; + scratch bytea; + i integer := 0; + bytecode_length integer := octet_length(bytecode); +BEGIN + WHILE i < bytecode_length LOOP + selected_byte := get_byte(bytecode, i); + IF selected_byte > 78 OR selected_byte = 0 THEN + -- OP_0 (0) and all opcodes after OP_PUSHDATA_4 (78) are single-byte instructions + i := i + 1; + maybe_redeem := NULL; + ELSIF selected_byte > 0 AND selected_byte <= 75 THEN + -- OP_PUSHBYTES_1 (1) through OP_PUSHBYTES_75 (75) directly indicate the length of pushed data + maybe_redeem := substring(bytecode from (i + 2) for selected_byte); + i := i + 1 + selected_byte; + ELSIF selected_byte = 76 THEN + -- OP_PUSHDATA_1 reads one length-byte + length_value := get_byte(bytecode, (i + 1)); + maybe_redeem := substring(bytecode from (i + 3) for length_value); + i := i + 2 + length_value; + ELSIF selected_byte = 77 THEN + -- OP_PUSHDATA_2 reads two length-bytes + scratch := substring(bytecode from (i + 2) for 2); + -- parse scratch as unsigned, two byte, little-endian number: + length_value := ((get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + maybe_redeem := substring(bytecode from (i + 4) for length_value); + i := i + 3 + length_value; + ELSIF selected_byte = 78 THEN + -- OP_PUSHDATA_4 reads four length-bytes + scratch := substring(bytecode from (i + 2) for 4); + -- parse scratch as unsigned, four byte, little-endian number: + length_value := ((get_byte(scratch, 3) << 24) | (get_byte(scratch, 2) << 16) | (get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + maybe_redeem := substring(bytecode from (i + 6) for length_value); + i := i + 5 + length_value; + END IF; + END LOOP; + IF maybe_redeem = NULL THEN + RETURN maybe_redeem; + ELSE + RETURN parse_bytecode_pattern(maybe_redeem); + END IF; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/up.sql b/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/up.sql new file mode 100644 index 00000000..982c46b5 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/up.sql @@ -0,0 +1,81 @@ +CREATE OR REPLACE FUNCTION parse_bytecode_pattern_redeem(bytecode bytea) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + maybe_redeem bytea; + selected_byte integer; + length_value integer; + scratch bytea; + i integer := 0; + bytecode_length integer := octet_length(bytecode); +BEGIN + WHILE i < bytecode_length LOOP + selected_byte := get_byte(bytecode, i); + IF selected_byte > 78 OR selected_byte = 0 THEN + -- OP_0 (0) and all opcodes after OP_PUSHDATA_4 (78) are single-byte instructions + i := i + 1; + maybe_redeem := NULL; + ELSIF selected_byte > 0 AND selected_byte <= 75 THEN + -- OP_PUSHBYTES_1 (1) through OP_PUSHBYTES_75 (75) directly indicate the length of pushed data + IF bytecode_length - i < 1 + selected_byte THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + maybe_redeem := substring(bytecode from (i + 2) for selected_byte); + i := i + 1 + selected_byte; + ELSIF selected_byte = 76 THEN + IF bytecode_length - i < 2 THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + -- OP_PUSHDATA_1 reads one length-byte + length_value := get_byte(bytecode, (i + 1)); + IF bytecode_length - i < 2 + length_value THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + maybe_redeem := substring(bytecode from (i + 3) for length_value); + i := i + 2 + length_value; + ELSIF selected_byte = 77 THEN + IF bytecode_length - i < 3 THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + -- OP_PUSHDATA_2 reads two length-bytes + scratch := substring(bytecode from (i + 2) for 2); + -- parse scratch as unsigned, two byte, little-endian number: + length_value := ((get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + IF bytecode_length - i < 3 + length_value THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + maybe_redeem := substring(bytecode from (i + 4) for length_value); + i := i + 3 + length_value; + ELSIF selected_byte = 78 THEN + IF bytecode_length - i < 5 THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + -- OP_PUSHDATA_4 reads four length-bytes + scratch := substring(bytecode from (i + 2) for 4); + IF get_byte(scratch, 3) << 24 < 0 THEN + -- push length exceeds maximum signed int (>2GB) + RETURN NULL; + END IF; + -- parse scratch as unsigned, four byte, little-endian number: + length_value := ((get_byte(scratch, 3) << 24) | (get_byte(scratch, 2) << 16) | (get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + IF bytecode_length - i - 5 < length_value THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + maybe_redeem := substring(bytecode from (i + 6) for length_value); + i := i + 5 + length_value; + END IF; + END LOOP; + IF maybe_redeem IS NULL THEN + RETURN maybe_redeem; + ELSE + RETURN parse_bytecode_pattern(maybe_redeem); + END IF; +END; +$$; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 102d0dcd..fd9234fb 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -79,6 +79,7 @@ const dbUpMigrationPaths = [ 'default/1778435997270_add_node_transaction_history_primary_key/up.sql' ), migration('default/1778437612917_fix_zero_length_pushdata_patterns/up.sql'), + migration('default/1778438318512_fix_redeem_bytecode_parser/up.sql'), ]; const chaingraphInternalApiPort = '3201'; From 48e3092982f66598998337726454874711efc15e Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 20:48:52 +0100 Subject: [PATCH 23/63] test(e2e): reproduce mempool transaction expiry bug (#73) --- src/e2e/e2e.spec.ts | 245 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index fd9234fb..154244f3 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -159,6 +159,8 @@ const e2eEnvVariables = { CHAINGRAPH_INTERNAL_API_PORT: chaingraphInternalApiPort, CHAINGRAPH_LOG_FIREHOSE: logP2pMessage.toString(), CHAINGRAPH_LOG_PATH: chaingraphE2eLogPath, + CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS: '1000', + CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS: '100', CHAINGRAPH_POSTGRES_CONNECTION_STRING: postgresE2eConnectionStringTestDb, CHAINGRAPH_TRUSTED_NODES: e2eTrustedNodesSet1, NODE_ENV: 'production', @@ -607,6 +609,69 @@ const waitForBlockTransactionCount = async ( ); }; +const mempoolExpirationPollingAttempts = 50; +const mempoolExpirationPollingIntervalMs = 100; +const getExpiredMempoolArchiveState = async () => + ( + await client.query<{ + historyRowCount: number; + inMempool: boolean; + replacedAt: string | null; + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')), + ('expiry_child_c', decode(repeat('d3', 32), 'hex')) +), +selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +SELECT named_transactions.name AS "transactionName", + (node_transaction.transaction_internal_id IS NOT NULL) AS "inMempool", + COUNT(node_transaction_history.transaction_internal_id)::integer AS "historyRowCount", + MIN(node_transaction_history.replaced_at)::text AS "replacedAt" + FROM named_transactions + CROSS JOIN selected_node + LEFT JOIN node_transaction + ON node_transaction.node_internal_id = selected_node.internal_id + AND node_transaction.transaction_internal_id = named_transactions.internal_id + LEFT JOIN node_transaction_history + ON node_transaction_history.node_internal_id = selected_node.internal_id + AND node_transaction_history.transaction_internal_id = named_transactions.internal_id + GROUP BY named_transactions.name, node_transaction.transaction_internal_id + ORDER BY named_transactions.name; +`) + ).rows; +const waitForExpiredMempoolArchive = async ( + remainingAttempts = mempoolExpirationPollingAttempts +): Promise>> => { + const rows = await getExpiredMempoolArchiveState(); + const expectedReplacedAt = '2026-01-01 00:00:01'; + if ( + rows.every( + (row) => + !row.inMempool && + row.historyRowCount === 1 && + row.replacedAt === expectedReplacedAt + ) || + remainingAttempts === 0 + ) { + return rows; + } + await sleep(mempoolExpirationPollingIntervalMs); + return waitForExpiredMempoolArchive(remainingAttempts - 1); +}; + test.serial( '[e2e] ignores inbound transactions before initial sync is complete', async (t) => { @@ -848,6 +913,186 @@ SELECT transaction_values.name AS "transactionName", } ); +test.serial( + '[e2e] archives expired mempool transactions and descendants', + async (t) => { + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')), + ('expiry_child_c', decode(repeat('d3', 32), 'hex')) +) +INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, 1, 0, 100, false + FROM transaction_values; +`); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')) +) +INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + SELECT hash, 0, 1000, '\\x51'::bytea + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH input_values (child_name, parent_name, input_index) AS ( + VALUES + ('expiry_child_b', 'expiry_parent_a', 0), + ('expiry_child_c', 'expiry_child_b', 0) +), +transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')), + ('expiry_child_c', decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id, transaction.hash + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT child.internal_id, input_values.input_index, 0, 0, parent.hash, '\\x51'::bytea + FROM input_values + JOIN named_transactions child + ON child.name = input_values.child_name + JOIN named_transactions parent + ON parent.name = input_values.parent_name; +`); + await client.query(/* sql */ ` +WITH selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')), + ('expiry_child_c', decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT selected_node.internal_id, + named_transactions.internal_id, + CASE + WHEN named_transactions.name = 'expiry_parent_a' + THEN timestamp '2026-01-01 00:00:00' + ELSE CURRENT_TIMESTAMP + interval '1 day' + END + FROM selected_node + CROSS JOIN named_transactions; +`); + const archivedTransactions = await waitForExpiredMempoolArchive(); + t.deepEqual(archivedTransactions, [ + { + historyRowCount: 1, + inMempool: false, + replacedAt: '2026-01-01 00:00:01', + transactionName: 'expiry_child_b', + }, + { + historyRowCount: 1, + inMempool: false, + replacedAt: '2026-01-01 00:00:01', + transactionName: 'expiry_child_c', + }, + { + historyRowCount: 1, + inMempool: false, + replacedAt: '2026-01-01 00:00:01', + transactionName: 'expiry_parent_a', + }, + ]); + } finally { + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM node_transaction + USING named_transactions + WHERE node_transaction.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM node_transaction_history + USING named_transactions + WHERE node_transaction_history.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM input + USING named_transactions + WHERE input.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +) +DELETE FROM output + USING transaction_values + WHERE output.transaction_hash = transaction_values.hash; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +) +DELETE FROM transaction + USING transaction_values + WHERE transaction.hash = transaction_values.hash; +`); + } + } +); + test.serial( '[e2e] backfills existing orphan mempool descendants with idempotence', async (t) => { From 6533d31abba683fc8b69accecfd99c01772603a6 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 21:13:59 +0100 Subject: [PATCH 24/63] fix(agent): expire stale mempool transactions (#73) --- defaults.env | 8 +++ src/agent.ts | 123 +++++++++++++++++++++++++++++++++++++++++++- src/config.ts | 30 +++++++++++ src/db.ts | 105 +++++++++++++++++++++++++++++++++++++ src/e2e/e2e.spec.ts | 9 ++-- 5 files changed, 269 insertions(+), 6 deletions(-) diff --git a/defaults.env b/defaults.env index 6ec898d2..efb79e71 100644 --- a/defaults.env +++ b/defaults.env @@ -23,6 +23,14 @@ CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB= # startup repair task. CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE=10000 +# BCHN expires transactions from the mempool after 336 hours but does not announce +# those expirations. At initial sync and periodically thereafter the agent scans for +# node_transaction rows expiring soon and schedules each row to be archived to +# node_transaction_history at its exact expiry time. The existing mempool +# invalidation cascade archives same-node descendants. See bitauth/chaingraph#73. +CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS=1209600000 +CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS=3600000 + # A mapping of network magic bytes to hex-encoded genesis blocks. # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated. # E.g. CHAINGRAPH_GENESIS_BLOCKS=e3e1f3e8:rawblockhex,deadbeef:rawblockhex diff --git a/src/agent.ts b/src/agent.ts index 61ca75fb..9326d984 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -32,15 +32,19 @@ import { chaingraphUserAgent, genesisBlocks, incompleteBlockRepairBatchSize, + mempoolTransactionExpirationMs, + mempoolTransactionExpirationScanIntervalMs, postgresMaxConnections, trustedNodes, } from './config.js'; import { acceptBlocksViaHeaders, + archiveMempoolTransaction, createIndexes, getAllKnownBlockHashes, getIncompleteBlocks, getIndexCreationProgress, + getMempoolTransactionsExpiringBefore, listExistingIndexes, optionallyDisableSynchronousCommit, optionallyEnableSynchronousCommit, @@ -52,7 +56,7 @@ import { saveBlock, saveTransactionForNodes, } from './db.js'; -import type { IncompleteBlock } from './db.js'; +import type { ExpiringMempoolTransaction, IncompleteBlock } from './db.js'; import type { ChaingraphBlock } from './types/chaingraph.js'; // eslint-disable-next-line @typescript-eslint/naming-convention @@ -304,6 +308,15 @@ export class Agent { incompleteBlockRepairTimeout: ReturnType | undefined; + mempoolTransactionExpirationScanTimeout: + | ReturnType + | undefined; + + mempoolTransactionExpirationTimers = new Map< + string, + ReturnType + >(); + incompleteBlockRepairNextHeight = 0; completedIncompleteBlockRepairScan = false; @@ -853,6 +866,7 @@ export class Agent { } this.logger.info('Agent: enabled mempool tracking.'); this.saveInboundTransactions = true; + this.scheduleMempoolTransactionExpirationScan(); this.scheduleIncompleteBlockRepair(); }); }) @@ -932,6 +946,106 @@ export class Agent { return indexCreationCompletion; } + canScanForMempoolTransactionExpirations() { + return ![ + !this.completedInitialSync, + !this.saveInboundTransactions, + this.mempoolTransactionExpirationScanTimeout !== undefined, + this.willShutdown, + ].includes(true); + } + + scheduleMempoolTransactionExpirationScan(delayMs = 0, forceSchedule = false) { + if ( + (!forceSchedule && !this.canScanForMempoolTransactionExpirations()) || + this.willShutdown + ) { + return; + } + this.mempoolTransactionExpirationScanTimeout = setTimeout(() => { + this.mempoolTransactionExpirationScanTimeout = undefined; + this.scanForMempoolTransactionExpirations() + .catch((err) => { + this.logger.error( + err, + 'Agent: failed to scan for expiring mempool transactions.' + ); + }) + .finally(() => { + this.scheduleMempoolTransactionExpirationScan( + mempoolTransactionExpirationScanIntervalMs, + true + ); + }); + }, delayMs); + } + + scheduleMempoolTransactionExpiration( + transaction: ExpiringMempoolTransaction + ) { + const key = `${transaction.nodeInternalId}:${transaction.transactionInternalId}`; + if (this.mempoolTransactionExpirationTimers.has(key)) { + return; + } + const delayMs = Math.max(transaction.expiresAt.getTime() - Date.now(), 0); + const timeout = setTimeout(() => { + this.mempoolTransactionExpirationTimers.delete(key); + this.expireMempoolTransaction(transaction).catch((err) => { + this.logger.error( + err, + `Agent: failed to expire mempool transaction ${transaction.hash} for node ${transaction.nodeName}.` + ); + }); + }, delayMs); + this.mempoolTransactionExpirationTimers.set(key, timeout); + } + + async scanForMempoolTransactionExpirations() { + if (!this.saveInboundTransactions || this.willShutdown) { + return; + } + const expiresBefore = new Date( + Date.now() + mempoolTransactionExpirationScanIntervalMs + ); + const expiringTransactions = await getMempoolTransactionsExpiringBefore({ + expirationMs: mempoolTransactionExpirationMs, + expiresBefore, + }); + if (expiringTransactions.length === 0) { + this.logger.debug( + `Agent: no mempool transactions expiring before ${expiresBefore.toISOString()}; next scan in ${mempoolTransactionExpirationScanIntervalMs.toLocaleString()}ms.` + ); + return; + } + this.logger.info( + `Agent: found ${ + expiringTransactions.length + } mempool transaction(s) expiring before ${expiresBefore.toISOString()}; scheduling exact expiry.` + ); + expiringTransactions.forEach((transaction) => { + this.scheduleMempoolTransactionExpiration(transaction); + }); + } + + async expireMempoolTransaction(transaction: ExpiringMempoolTransaction) { + const archivedCount = await archiveMempoolTransaction({ + nodeInternalId: transaction.nodeInternalId, + replacedAt: transaction.expiresAt, + transactionInternalId: transaction.transactionInternalId, + }); + if (archivedCount === 0) { + this.logger.debug( + `Agent: skipped expiry of mempool transaction ${transaction.hash} for node ${transaction.nodeName}; it has already left node_transaction.` + ); + return; + } + this.logger.warn( + `Agent: expired mempool transaction ${transaction.hash} for node ${ + transaction.nodeName + }; archived from node_transaction to node_transaction_history with replaced_at ${transaction.expiresAt.toISOString()}.` + ); + } + canScheduleIncompleteBlockRepair() { return ![ incompleteBlockRepairBatchSize === 0, @@ -1980,6 +2094,13 @@ export class Agent { if (this.incompleteBlockRepairTimeout !== undefined) { clearTimeout(this.incompleteBlockRepairTimeout); } + if (this.mempoolTransactionExpirationScanTimeout !== undefined) { + clearTimeout(this.mempoolTransactionExpirationScanTimeout); + } + this.mempoolTransactionExpirationTimers.forEach((timeout) => { + clearTimeout(timeout); + }); + this.mempoolTransactionExpirationTimers.clear(); Object.values(this.nodes).forEach((connection) => { connection.disconnect(); }); diff --git a/src/config.ts b/src/config.ts index 76435f18..00c37a5a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -42,6 +42,8 @@ const expectedOptions = [ 'CHAINGRAPH_LOG_LEVEL_STDOUT', 'CHAINGRAPH_LOG_LEVEL_PATH', 'CHAINGRAPH_LOG_PATH', + 'CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS', + 'CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS', 'CHAINGRAPH_POSTGRES_CONNECTION_STRING', 'CHAINGRAPH_POSTGRES_MAX_CONNECTIONS', 'CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT', @@ -126,6 +128,32 @@ if ( ); } +const mempoolTransactionExpirationMs = Number( + configuration.CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS +); +if ( + !Number.isInteger(mempoolTransactionExpirationMs) || + mempoolTransactionExpirationMs <= 0 +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + 'The CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS environment variable must be an integer greater than 0.' + ); +} + +const mempoolTransactionExpirationScanIntervalMs = Number( + configuration.CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS +); +if ( + !Number.isInteger(mempoolTransactionExpirationScanIntervalMs) || + mempoolTransactionExpirationScanIntervalMs <= 0 +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + 'The CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS environment variable must be an integer greater than 0.' + ); +} + const extendTildeAndResolvePath = (path: string) => path.startsWith('~') ? resolve(join(homedir(), path.slice(1))) @@ -391,6 +419,8 @@ export { chaingraphUserAgent, genesisBlocks, incompleteBlockRepairBatchSize, + mempoolTransactionExpirationMs, + mempoolTransactionExpirationScanIntervalMs, postgresMaxConnections, postgresConnectionString, postgresSynchronousCommit, diff --git a/src/db.ts b/src/db.ts index 34ba88c3..f13050c0 100644 --- a/src/db.ts +++ b/src/db.ts @@ -106,6 +106,15 @@ export interface IncompleteBlockScan { scannedBlockCount: number; } +export interface ExpiringMempoolTransaction { + expiresAt: Date; + hash: string; + nodeInternalId: number; + nodeName: string; + transactionInternalId: number; + validatedAt: Date; +} + /** * Find blocks for which the locally saved block_transaction rows don't sum to * the block's saved byte size. This avoids the SQL block encoder so it can @@ -237,6 +246,102 @@ SELECT } }; +/** + * Find node_transaction rows which will expire before the provided timestamp. + */ +export const getMempoolTransactionsExpiringBefore = async ({ + expiresBefore, + expirationMs, +}: { + expirationMs: number; + expiresBefore: Date; +}): Promise => { + const expirationInterval = `${expirationMs}::double precision * interval '1 millisecond'`; + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const transactions = await client.query<{ + expiresAt: string; + hash: string; + nodeInternalId: string; + nodeName: string; + transactionInternalId: string; + validatedAt: string; + }>(/* sql */ ` +SELECT encode(transaction.hash, 'hex') AS "hash", + node.name AS "nodeName", + node_transaction.node_internal_id AS "nodeInternalId", + node_transaction.transaction_internal_id AS "transactionInternalId", + node_transaction.validated_at::text AS "validatedAt", + (node_transaction.validated_at + (${expirationInterval}))::text AS "expiresAt" + FROM node_transaction + JOIN node + ON node.internal_id = node_transaction.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + WHERE node_transaction.validated_at + (${expirationInterval}) <= ${dateToTimestampWithoutTimezone( + expiresBefore + )} + ORDER BY "expiresAt", "nodeName", "hash"; +`); + return transactions.rows.map((transaction) => ({ + expiresAt: timestampWithoutTimezoneToDate(transaction.expiresAt), + hash: transaction.hash, + nodeInternalId: Number(transaction.nodeInternalId), + nodeName: transaction.nodeName, + transactionInternalId: Number(transaction.transactionInternalId), + validatedAt: timestampWithoutTimezoneToDate(transaction.validatedAt), + })); + } finally { + client.release(); + } +}; + +/** + * Archive a single node_transaction row. Existing history triggers handle any + * same-node descendants with the same replaced_at timestamp. + */ +export const archiveMempoolTransaction = async ({ + nodeInternalId, + replacedAt, + transactionInternalId, +}: { + nodeInternalId: number; + replacedAt: Date; + transactionInternalId: number; +}) => { + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const result = await client.query<{ + archivedCount: number; + }>( + /* sql */ ` +WITH deleted_row AS ( + DELETE FROM node_transaction + WHERE node_internal_id = $1 + AND transaction_internal_id = $2 + RETURNING node_internal_id, + transaction_internal_id, + validated_at, + ${dateToTimestampWithoutTimezone(replacedAt)} AS replaced_at +), +inserted_history AS ( + INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at + FROM deleted_row + RETURNING transaction_internal_id +) +SELECT COUNT(*)::integer AS "archivedCount" FROM inserted_history; +`, + [nodeInternalId, transactionInternalId] + ); + return result.rows[0]!.archivedCount; + } finally { + client.release(); + } +}; + /** * Create or update one or more trusted node in the Chaingraph database, * returning it's internal ID. diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 154244f3..3cbba71c 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -159,7 +159,6 @@ const e2eEnvVariables = { CHAINGRAPH_INTERNAL_API_PORT: chaingraphInternalApiPort, CHAINGRAPH_LOG_FIREHOSE: logP2pMessage.toString(), CHAINGRAPH_LOG_PATH: chaingraphE2eLogPath, - CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS: '1000', CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS: '100', CHAINGRAPH_POSTGRES_CONNECTION_STRING: postgresE2eConnectionStringTestDb, CHAINGRAPH_TRUSTED_NODES: e2eTrustedNodesSet1, @@ -656,7 +655,7 @@ const waitForExpiredMempoolArchive = async ( remainingAttempts = mempoolExpirationPollingAttempts ): Promise>> => { const rows = await getExpiredMempoolArchiveState(); - const expectedReplacedAt = '2026-01-01 00:00:01'; + const expectedReplacedAt = '2026-01-15 00:00:00'; if ( rows.every( (row) => @@ -999,19 +998,19 @@ INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validat { historyRowCount: 1, inMempool: false, - replacedAt: '2026-01-01 00:00:01', + replacedAt: '2026-01-15 00:00:00', transactionName: 'expiry_child_b', }, { historyRowCount: 1, inMempool: false, - replacedAt: '2026-01-01 00:00:01', + replacedAt: '2026-01-15 00:00:00', transactionName: 'expiry_child_c', }, { historyRowCount: 1, inMempool: false, - replacedAt: '2026-01-01 00:00:01', + replacedAt: '2026-01-15 00:00:00', transactionName: 'expiry_parent_a', }, ]); From 1f32bc0ecaab1154e1a9421844807eb29c90174f Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 22:23:46 +0100 Subject: [PATCH 25/63] fix(agent): archive confirmed txns stuck in mempool table before processing expiry (#73) --- src/agent.ts | 39 +++++++- src/db.ts | 111 +++++++++++++++++++++ src/e2e/e2e.spec.ts | 238 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 387 insertions(+), 1 deletion(-) diff --git a/src/agent.ts b/src/agent.ts index 9326d984..e830206b 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -40,6 +40,7 @@ import { import { acceptBlocksViaHeaders, archiveMempoolTransaction, + archiveMempoolTransactionsAcceptedByBlocks, createIndexes, getAllKnownBlockHashes, getIncompleteBlocks, @@ -866,8 +867,8 @@ export class Agent { } this.logger.info('Agent: enabled mempool tracking.'); this.saveInboundTransactions = true; - this.scheduleMempoolTransactionExpirationScan(); this.scheduleIncompleteBlockRepair(); + this.scheduleMempoolTransactionExpirationScan(); }); }) .catch((err) => { @@ -950,11 +951,19 @@ export class Agent { return ![ !this.completedInitialSync, !this.saveInboundTransactions, + this.waitingForIncompleteBlockRepairScan(), this.mempoolTransactionExpirationScanTimeout !== undefined, this.willShutdown, ].includes(true); } + waitingForIncompleteBlockRepairScan() { + return ( + incompleteBlockRepairBatchSize !== 0 && + !this.completedIncompleteBlockRepairScan + ); + } + scheduleMempoolTransactionExpirationScan(delayMs = 0, forceSchedule = false) { if ( (!forceSchedule && !this.canScanForMempoolTransactionExpirations()) || @@ -1004,6 +1013,7 @@ export class Agent { if (!this.saveInboundTransactions || this.willShutdown) { return; } + await this.archiveAcceptedMempoolTransactions(); const expiresBefore = new Date( Date.now() + mempoolTransactionExpirationScanIntervalMs ); @@ -1046,6 +1056,32 @@ export class Agent { ); } + async archiveAcceptedMempoolTransactions() { + const archivedTransactions = + await archiveMempoolTransactionsAcceptedByBlocks(); + if (archivedTransactions.length === 0) { + return; + } + this.logger.warn( + `Agent: archived ${archivedTransactions.length.toLocaleString()} stale mempool transaction(s) already accepted or replaced by blocks before ordinary expiry.` + ); + archivedTransactions.forEach((transaction) => { + if (transaction.replacedAt === null) { + this.logger.info( + `Agent: archived stale confirmed mempool transaction ${transaction.hash} for node ${transaction.nodeName}; transaction is already accepted by a block, archived with replaced_at NULL.` + ); + return; + } + this.logger.info( + `Agent: archived stale replaced mempool transaction ${ + transaction.hash + } for node ${ + transaction.nodeName + }; an accepted block already spends the same outpoint, archived with replaced_at ${transaction.replacedAt.toISOString()}.` + ); + }); + } + canScheduleIncompleteBlockRepair() { return ![ incompleteBlockRepairBatchSize === 0, @@ -1110,6 +1146,7 @@ export class Agent { this.incompleteBlockRepairNextHeight = 0; this.completedIncompleteBlockRepairScan = true; this.logger.info('Agent: completed incomplete block repair scan.'); + this.scheduleMempoolTransactionExpirationScan(); return; } this.incompleteBlockRepairNextHeight = heightUpperBound; diff --git a/src/db.ts b/src/db.ts index f13050c0..58a1f62f 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ import pg from 'pg'; import type { Agent } from './agent.js'; @@ -115,6 +116,12 @@ export interface ExpiringMempoolTransaction { validatedAt: Date; } +export interface ArchivedMempoolTransaction { + hash: string; + nodeName: string; + replacedAt: Date | null; +} + /** * Find blocks for which the locally saved block_transaction rows don't sum to * the block's saved byte size. This avoids the SQL block encoder so it can @@ -297,6 +304,110 @@ SELECT encode(transaction.hash, 'hex') AS "hash", } }; +/** + * Archive node_transaction rows for transactions that are already accepted or + * replaced by accepted blocks for the same node. This repairs historical rows + * missed when block inclusions are added after the node_block trigger has + * already fired. + */ +export const archiveMempoolTransactionsAcceptedByBlocks = async (): Promise< + ArchivedMempoolTransaction[] +> => { + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const result = await client.query<{ + hash: string; + nodeName: string; + replacedAt: string | null; + }>(/* sql */ ` +WITH directly_accepted AS ( + SELECT node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + NULL::timestamp without time zone AS replaced_at + FROM node_transaction + JOIN block_transaction + ON block_transaction.transaction_internal_id = node_transaction.transaction_internal_id + JOIN node_block + ON node_block.node_internal_id = node_transaction.node_internal_id + AND node_block.block_internal_id = block_transaction.block_internal_id +), +replaced_by_accepted AS ( + SELECT node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + MIN(node_block.accepted_at) AS replaced_at + FROM node_transaction + JOIN input mempool_input + ON mempool_input.transaction_internal_id = node_transaction.transaction_internal_id + JOIN input accepted_input + ON accepted_input.outpoint_transaction_hash = mempool_input.outpoint_transaction_hash + AND accepted_input.outpoint_index = mempool_input.outpoint_index + AND accepted_input.transaction_internal_id != node_transaction.transaction_internal_id + JOIN block_transaction + ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id + JOIN node_block + ON node_block.node_internal_id = node_transaction.node_internal_id + AND node_block.block_internal_id = block_transaction.block_internal_id + WHERE mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea + GROUP BY node_transaction.node_internal_id, + node_transaction.transaction_internal_id +), +archive_candidates AS ( + SELECT node_internal_id, transaction_internal_id, replaced_at + FROM directly_accepted + UNION ALL + SELECT node_internal_id, transaction_internal_id, replaced_at + FROM replaced_by_accepted +), +archive_rows AS ( + SELECT node_internal_id, + transaction_internal_id, + CASE + WHEN bool_or(replaced_at IS NULL) THEN NULL::timestamp without time zone + ELSE MIN(replaced_at) + END AS replaced_at + FROM archive_candidates + GROUP BY node_internal_id, transaction_internal_id +), +deleted_rows AS ( + DELETE FROM node_transaction + USING archive_rows + WHERE node_transaction.node_internal_id = archive_rows.node_internal_id + AND node_transaction.transaction_internal_id = archive_rows.transaction_internal_id + RETURNING node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + node_transaction.validated_at, + archive_rows.replaced_at +), +inserted_history AS ( + INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at + FROM deleted_rows + RETURNING node_internal_id, transaction_internal_id, replaced_at +) +SELECT encode(transaction.hash, 'hex') AS "hash", + node.name AS "nodeName", + inserted_history.replaced_at::text AS "replacedAt" + FROM inserted_history + JOIN node + ON node.internal_id = inserted_history.node_internal_id + JOIN transaction + ON transaction.internal_id = inserted_history.transaction_internal_id + ORDER BY "nodeName", "hash"; +`); + return result.rows.map((row) => ({ + hash: row.hash, + nodeName: row.nodeName, + replacedAt: + row.replacedAt === null + ? null + : timestampWithoutTimezoneToDate(row.replacedAt), + })); + } finally { + client.release(); + } +}; + /** * Archive a single node_transaction row. Existing history triggers handle any * same-node descendants with the same replaced_at timestamp. diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 3cbba71c..f3c43ca3 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -671,6 +671,57 @@ const waitForExpiredMempoolArchive = async ( return waitForExpiredMempoolArchive(remainingAttempts - 1); }; +const getConfirmedMempoolArchiveState = async () => + ( + await client.query<{ + historyRowCount: number; + inMempool: boolean; + replacedAt: string | null; + }>(/* sql */ ` +WITH selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +) +SELECT (node_transaction.transaction_internal_id IS NOT NULL) AS "inMempool", + COUNT(node_transaction_history.transaction_internal_id)::integer AS "historyRowCount", + MIN(node_transaction_history.replaced_at)::text AS "replacedAt" + FROM selected_transaction + CROSS JOIN selected_node + LEFT JOIN node_transaction + ON node_transaction.node_internal_id = selected_node.internal_id + AND node_transaction.transaction_internal_id = selected_transaction.internal_id + LEFT JOIN node_transaction_history + ON node_transaction_history.node_internal_id = selected_node.internal_id + AND node_transaction_history.transaction_internal_id = selected_transaction.internal_id + GROUP BY node_transaction.transaction_internal_id; +`) + ).rows[0]; + +const confirmedMempoolArchiveCompleted = ( + row: Awaited> +) => + row !== undefined && + !row.inMempool && + row.historyRowCount === 1 && + row.replacedAt === null; + +const waitForConfirmedMempoolArchive = async ( + remainingAttempts = mempoolExpirationPollingAttempts +): Promise>> => { + const row = await getConfirmedMempoolArchiveState(); + if (confirmedMempoolArchiveCompleted(row) || remainingAttempts === 0) { + return row; + } + await sleep(mempoolExpirationPollingIntervalMs); + return waitForConfirmedMempoolArchive(remainingAttempts - 1); +}; + test.serial( '[e2e] ignores inbound transactions before initial sync is complete', async (t) => { @@ -1092,6 +1143,193 @@ DELETE FROM transaction } ); +test.serial( + '[e2e] archives stale mempool transactions already accepted by blocks', + async (t) => { + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('confirmed_parent_a', decode(repeat('d4', 32), 'hex')), + ('confirmed_child_b', decode(repeat('d5', 32), 'hex')) +) +INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, 1, 0, 100, false + FROM transaction_values; +`); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query(/* sql */ ` +INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + VALUES (decode(repeat('d4', 32), 'hex'), 0, 1000, '\\x51'::bytea); +`); + await client.query(/* sql */ ` +WITH selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +) +INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT selected_transaction.internal_id, + 0, + 0, + 0, + decode(repeat('d4', 32), 'hex'), + '\\x51'::bytea + FROM selected_transaction; +`); + await client.query(/* sql */ ` +WITH selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +), +inserted_block AS ( + INSERT INTO block (height, version, "timestamp", hash, previous_block_hash, merkle_root, bits, nonce, size_bytes) + VALUES (4001, 1, 0, decode(repeat('d6', 32), 'hex'), decode(repeat('d7', 32), 'hex'), decode(repeat('d8', 32), 'hex'), 0, 0, 181) + RETURNING internal_id +) +INSERT INTO block_transaction (block_internal_id, transaction_internal_id, transaction_index) + SELECT inserted_block.internal_id, selected_transaction.internal_id, 1 + FROM inserted_block + CROSS JOIN selected_transaction; +`); + await client.query(/* sql */ ` +WITH selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +selected_block AS ( + SELECT internal_id + FROM block + WHERE hash = decode(repeat('d6', 32), 'hex') +) +INSERT INTO node_block (node_internal_id, block_internal_id, accepted_at) + SELECT selected_node.internal_id, + selected_block.internal_id, + timestamp '2026-01-01 00:10:00' + FROM selected_node + CROSS JOIN selected_block; +`); + await client.query(/* sql */ ` +WITH selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +) +INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT selected_node.internal_id, + selected_transaction.internal_id, + timestamp '2026-01-01 00:00:00' + FROM selected_node + CROSS JOIN selected_transaction; +`); + const archivedTransaction = await waitForConfirmedMempoolArchive(); + t.deepEqual(archivedTransaction, { + historyRowCount: 1, + inMempool: false, + replacedAt: null, + }); + } finally { + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d4', 32), 'hex')), + (decode(repeat('d5', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM node_transaction + USING named_transactions + WHERE node_transaction.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d4', 32), 'hex')), + (decode(repeat('d5', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM node_transaction_history + USING named_transactions + WHERE node_transaction_history.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH selected_block AS ( + SELECT internal_id + FROM block + WHERE hash = decode(repeat('d6', 32), 'hex') +) +DELETE FROM node_block + USING selected_block + WHERE node_block.block_internal_id = selected_block.internal_id; +`); + await client.query(/* sql */ ` +WITH selected_block AS ( + SELECT internal_id + FROM block + WHERE hash = decode(repeat('d6', 32), 'hex') +) +DELETE FROM node_block_history + USING selected_block + WHERE node_block_history.block_internal_id = selected_block.internal_id; +`); + await client.query(/* sql */ ` +WITH selected_block AS ( + SELECT internal_id + FROM block + WHERE hash = decode(repeat('d6', 32), 'hex') +) +DELETE FROM block_transaction + USING selected_block + WHERE block_transaction.block_internal_id = selected_block.internal_id; +`); + await client.query(/* sql */ ` +DELETE FROM block + WHERE hash = decode(repeat('d6', 32), 'hex'); +`); + await client.query(/* sql */ ` +WITH selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +) +DELETE FROM input + USING selected_transaction + WHERE input.transaction_internal_id = selected_transaction.internal_id; +`); + await client.query(/* sql */ ` +DELETE FROM output + WHERE transaction_hash = decode(repeat('d4', 32), 'hex'); +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d4', 32), 'hex')), + (decode(repeat('d5', 32), 'hex')) +) +DELETE FROM transaction + USING transaction_values + WHERE transaction.hash = transaction_values.hash; +`); + } + } +); + test.serial( '[e2e] backfills existing orphan mempool descendants with idempotence', async (t) => { From 12402f80e7b72c0464b4421fe91b8af37717ca38 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 23:06:20 +0100 Subject: [PATCH 26/63] fix(agent): DatabaseError: null value violates not-null constraint --- src/db.ts | 60 ++++++++++----- src/e2e/e2e.spec.ts | 184 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+), 19 deletions(-) diff --git a/src/db.ts b/src/db.ts index 58a1f62f..fa4465e5 100644 --- a/src/db.ts +++ b/src/db.ts @@ -560,15 +560,6 @@ WITH transaction_values (hash, version, locktime, size_bytes, is_coinbase) AS ( )}'::bytea, '${hexToByteaString(input.unlockingBytecode)}'::bytea)` ) .join(',')} -), node_transaction_values (node_internal_id, validated_at) AS ( - VALUES ${nodeValidations - .map( - (validation) => - `(${ - validation.nodeInternalId - }::bigint, ${dateToTimestampWithoutTimezone(validation.validatedAt)})` - ) - .join(',')} ), new_transaction (transaction_hash, transaction_internal_id) AS ( INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) SELECT hash, version, locktime, size_bytes, is_coinbase FROM transaction_values @@ -580,21 +571,52 @@ WITH transaction_values (hash, version, locktime, size_bytes, is_coinbase) AS ( ), insert_inputs AS ( INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) SELECT transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode FROM input_values CROSS JOIN new_transaction -), new_or_existing_transaction (transaction_internal_id) AS ( - SELECT COALESCE ( - (SELECT transaction_internal_id FROM new_transaction), - (SELECT internal_id AS transaction_internal_id FROM transaction WHERE transaction.hash = '${hexToByteaString( - transaction.hash - )}'::bytea) - ) +) +SELECT COUNT(*) FROM new_transaction; +`; + const saveNodeValidations = /* sql */ ` +WITH node_transaction_values (node_internal_id, validated_at) AS ( + VALUES ${nodeValidations + .map( + (validation) => + `(${ + validation.nodeInternalId + }::bigint, ${dateToTimestampWithoutTimezone(validation.validatedAt)})` + ) + .join(',')} ) INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) - SELECT node_internal_id, transaction_internal_id, validated_at FROM node_transaction_values CROSS JOIN new_or_existing_transaction + SELECT node_internal_id, $1::bigint, validated_at FROM node_transaction_values ON CONFLICT ON CONSTRAINT "node_transaction_pkey" DO NOTHING; `; const client = await pool.connect(); - await client.query(saveTransaction); - client.release(); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query('BEGIN;'); + await client.query(saveTransaction); + const transactionInternalIdResult = await client.query<{ + internalId: string; + }>( + /* sql */ `SELECT internal_id AS "internalId" FROM transaction WHERE hash = $1;`, + [Buffer.from(transaction.hash, 'hex')] + ); + const transactionInternalId = + transactionInternalIdResult.rows[0]?.internalId; + if (transactionInternalId === undefined) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Failed to save or find transaction while recording node validation: ${transaction.hash}` + ); + } + await client.query(saveNodeValidations, [transactionInternalId]); + await client.query('COMMIT;'); + } catch (err) { + await client.query('ROLLBACK;'); + // eslint-disable-next-line functional/no-throw-statement + throw err; + } finally { + client.release(); + } }; /** diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index f3c43ca3..215a7aa9 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -27,6 +27,8 @@ import { execa } from 'execa'; import got from 'got'; import pg from 'pg'; +import type { ChaingraphTransaction } from '../types/chaingraph.js'; + import { chaingraphE2eLogPath, logger } from './e2e.spec.logging.helper.js'; import { chipnetCashTokensTx, @@ -576,6 +578,41 @@ const sleep = async (ms: number) => new Promise((res) => { setTimeout(res, ms); }); + +const repeatedHashByteLength = 32; +const transactionSaveConflictPollingAttempts = 50; +const transactionSaveConflictPollingIntervalMs = 20; +const waitForTransactionSaveConflict = async ( + transactionHash: string, + remainingAttempts = transactionSaveConflictPollingAttempts +): Promise => { + const result = await client.query<{ waiting: boolean }>( + /* sql */ ` + SELECT EXISTS ( + SELECT 1 + FROM pg_stat_activity + -- cspell:disable-next-line + WHERE datname = $1 + AND query LIKE '%INSERT INTO transaction%' + AND query LIKE $2 + AND wait_event_type IS NOT NULL + ) AS waiting; + `, + [e2eTestDbName, `%${transactionHash}%`] + ); + if (result.rows[0]!.waiting) { + return; + } + if (remainingAttempts === 0) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Timed out waiting for saveTransactionForNodes conflict on ${transactionHash}.` + ); + } + await sleep(transactionSaveConflictPollingIntervalMs); + await waitForTransactionSaveConflict(transactionHash, remainingAttempts - 1); +}; + const blockRepairPollingAttempts = 40; const blockRepairPollingIntervalMs = 250; const getBlockTransactionCount = async (blockHash: string) => @@ -806,6 +843,153 @@ test.serial('[e2e] creates expected indexes after initial sync', async (t) => { t.pass(); }); +test.serial( + '[e2e] records node validation after concurrent transaction insert conflict', + async (t) => { + const transactionHash = 'c1'.repeat(repeatedHashByteLength); + const validatedAt = new Date('2026-01-01T00:00:00.000Z'); + const transaction: ChaingraphTransaction = { + hash: transactionHash, + inputs: [ + { + outpointIndex: 0, + outpointTransactionHash: 'c2'.repeat(repeatedHashByteLength), + sequenceNumber: 0, + unlockingBytecode: '51', + }, + ], + isCoinbase: false, + locktime: 0, + outputs: [ + { + lockingBytecode: '51', + valueSatoshis: 1000n, + }, + ], + sizeBytes: 100, + version: 1, + }; + const nodeInternalId = Number( + ( + await client.query<{ internalId: number }>( + /* sql */ `SELECT internal_id AS "internalId" FROM node WHERE name = 'node1';` + ) + ).rows[0]!.internalId + ); + await client.query( + /* sql */ ` + DELETE FROM node_transaction + USING transaction + WHERE node_transaction.transaction_internal_id = transaction.internal_id + AND transaction.hash = $1; + `, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query( + /* sql */ ` + DELETE FROM input + USING transaction + WHERE input.transaction_internal_id = transaction.internal_id + AND transaction.hash = $1; + `, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query( + /* sql */ `DELETE FROM output WHERE transaction_hash = $1;`, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query(/* sql */ `DELETE FROM transaction WHERE hash = $1;`, [ + Buffer.from(transactionHash, 'hex'), + ]); + const originalPostgresConnectionString = + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + postgresE2eConnectionStringTestDb; + const { pool: dbPool, saveTransactionForNodes } = await import('../db.js'); + const competingClient = new pg.Client({ + connectionString: postgresE2eConnectionStringTestDb, + }); + await competingClient.connect(); + // eslint-disable-next-line functional/no-let + let competingTransactionOpen = false; + // eslint-disable-next-line functional/no-try-statement + try { + await competingClient.query(/* sql */ `BEGIN;`); + competingTransactionOpen = true; + await competingClient.query( + /* sql */ ` + INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + VALUES ($1, 1, 0, 100, false); + `, + [Buffer.from(transactionHash, 'hex')] + ); + const savePromise = saveTransactionForNodes(transaction, [ + { nodeInternalId, validatedAt }, + ]); + await waitForTransactionSaveConflict(transactionHash); + await competingClient.query(/* sql */ `COMMIT;`); + competingTransactionOpen = false; + await t.notThrowsAsync(savePromise); + const savedValidationCount = Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*)::bigint AS count + FROM node_transaction + JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + WHERE transaction.hash = $1 + AND node_transaction.node_internal_id = $2 + AND node_transaction.validated_at = $3; + `, + [Buffer.from(transactionHash, 'hex'), nodeInternalId, validatedAt] + ) + ).rows[0]!.count + ); + t.deepEqual(savedValidationCount, 1); + } finally { + if (competingTransactionOpen) { + await competingClient.query(/* sql */ `ROLLBACK;`).catch((err) => { + logger.debug(err); + }); + } + await client.query( + /* sql */ ` + DELETE FROM node_transaction + USING transaction + WHERE node_transaction.transaction_internal_id = transaction.internal_id + AND transaction.hash = $1; + `, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query( + /* sql */ ` + DELETE FROM input + USING transaction + WHERE input.transaction_internal_id = transaction.internal_id + AND transaction.hash = $1; + `, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query( + /* sql */ `DELETE FROM output WHERE transaction_hash = $1;`, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query(/* sql */ `DELETE FROM transaction WHERE hash = $1;`, [ + Buffer.from(transactionHash, 'hex'), + ]); + await competingClient.end(); + await dbPool.end(); + if (originalPostgresConnectionString === undefined) { + delete process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + } else { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + originalPostgresConnectionString; + } + } + } +); + test.serial( '[e2e] cascades replaced mempool transaction history to same-node descendants', async (t) => { From 3b1684f1f7826a5f1f6f9d11b2e7138944034df1 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 10 May 2026 23:47:00 +0100 Subject: [PATCH 27/63] fix(sql): add token category index --- src/agent.ts | 1 + src/components/db-utils.ts | 1 + src/e2e/e2e.spec.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/src/agent.ts b/src/agent.ts index e830206b..b7227440 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -360,6 +360,7 @@ export class Agent { 'block_inclusions_index', 'output_search_index', 'spent_by_index', + 'token_category_index', ]; /** diff --git a/src/components/db-utils.ts b/src/components/db-utils.ts index dfdacd0e..e1f03e51 100644 --- a/src/components/db-utils.ts +++ b/src/components/db-utils.ts @@ -4,6 +4,7 @@ export const indexDefinitions = { block_inclusions_index: /* sql */ `CREATE INDEX block_inclusions_index ON block_transaction USING btree (transaction_internal_id);`, output_search_index: /* sql */ `CREATE INDEX output_search_index ON output USING btree (substring(locking_bytecode, 0, 26));`, spent_by_index: /* sql */ `CREATE INDEX spent_by_index ON input USING btree (outpoint_transaction_hash, outpoint_index);`, + token_category_index: /* sql */ `CREATE INDEX token_category_index ON output USING btree (token_category);`, }; /* eslint-enable camelcase, @typescript-eslint/naming-convention */ diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 215a7aa9..8c8034b5 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -815,6 +815,7 @@ test.serial('[e2e] creates expected indexes after initial sync', async (t) => { 'output_pkey', 'output_search_index', 'spent_by_index', + 'token_category_index', 'transaction_hash_key', 'transaction_pkey', ]); From 61995041dfd8bf01af0b036d4c8ae60188eccf3a Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 11 May 2026 08:31:58 +0100 Subject: [PATCH 28/63] feat(agent): log db insert times --- src/agent.ts | 51 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index b7227440..2dbe80ed 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -116,6 +116,27 @@ const renderSyncPercentage = (value: number) => { }; const msPerSecond = 1000; +const durationDecimalPlaces = 1; +const transactionRateDecimalPlaces = 2; + +const formatDurationSeconds = (durationMs: number) => + `${(durationMs / msPerSecond).toFixed(durationDecimalPlaces)}s`; + +const formatTransactionRate = (transactionCount: number, durationMs: number) => + durationMs === 0 + ? 'n/a tx/s' + : `${((transactionCount * msPerSecond) / durationMs).toFixed( + transactionRateDecimalPlaces + )}tx/s`; + +const formatTransactionDuration = ( + transactionCount: number, + durationMs: number +) => + transactionCount === 0 + ? 'n/a/tx' + : `${formatDurationSeconds(durationMs / transactionCount)}/tx`; + /** * Convert a bitcoin block header timestamp (UTC in seconds) to a `Date`. * @param timestamp - the block header timestamp @@ -1736,6 +1757,7 @@ export class Agent { const durationMs = completionTime - startTime; const transactions = attemptedSavedTransactions.length; + const savedTransactionCount = transactions - transactionCacheMisses; const inputs = attemptedSavedTransactions.reduce( (total, tx) => total + tx.inputs.length, 0 @@ -1769,15 +1791,29 @@ export class Agent { .toString() .padStart(heightMinWidth, ' ')} | timestamp: ${blockTimestampToDate( block.timestamp - ).toISOString()} | hash: ${block.hash} | new txs: ${ - transactions - transactionCacheMisses - }/${block.transactions.length.toString()} (${transactionCacheMisses} cache misses) | nodes: ${nodeAcceptances + ).toISOString()} | hash: ${ + block.hash + } | new txs: ${savedTransactionCount}/${block.transactions.length.toString()} (${transactionCacheMisses} cache misses) | nodes: ${nodeAcceptances .map((acceptance) => acceptance.nodeName) .join(', ')}`; + const blockInsertLog = `Inserting block ${ + block.height + } with ${savedTransactionCount} new transaction${ + savedTransactionCount === 1 ? '' : 's' + } for ${nodeAcceptances + .map((acceptance) => acceptance.nodeName) + .join(', ')} took ${formatDurationSeconds( + durationMs + )} (${formatTransactionDuration( + savedTransactionCount, + durationMs + )}, ${formatTransactionRate(savedTransactionCount, durationMs)})`; if (isHistoricalSync) { this.logger.debug(blockSyncLog); + this.logger.trace(blockInsertLog); } else { this.logger.info(blockSyncLog); + this.logger.debug(blockInsertLog); } nodeAcceptances.forEach((acceptance) => { @@ -1875,7 +1911,7 @@ export class Agent { txCacheItem.db = true; this.transactionCache.set(transactionHash, txCacheItem); this.logger.trace( - `Marked transaction saved to DB - hash: ${transactionHash}` + `Marked transaction as saved to DB - hash: ${transactionHash}` ); } @@ -1917,7 +1953,14 @@ export class Agent { .join(', ')} - hash: ${tx.hash}` ); // TODO: collect statistics on save speed + const startTime = Date.now(); await saveTransactionForNodes(tx, validations); + const durationMs = Date.now() - startTime; + this.logger.debug( + `Inserting mempool tx ${ + tx.hash + } for node ${nodeName} took ${formatDurationSeconds(durationMs)}` + ); this.markTransactionSavedToDb(tx.hash); } } From 1f75cea8496f355e8673e15646ee35aa9aad6c0e Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 13 May 2026 10:35:00 +0100 Subject: [PATCH 29/63] test(agent): reproduce slow header catch-up sync state path --- src/components/sync-state.spec.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/components/sync-state.spec.ts b/src/components/sync-state.spec.ts index b21a1f54..634bb7b1 100644 --- a/src/components/sync-state.spec.ts +++ b/src/components/sync-state.spec.ts @@ -58,3 +58,29 @@ test('SyncState', (t) => { t.deepEqual(state.additionalSyncedHeights, []); t.deepEqual(state.latestSyncedBlockTime, new Date(5)); }); + +test('SyncState scales when marking many contiguous heights as pending', (t) => { + const state = new SyncState({ + additionalSyncedHeights: [], + fullySyncedUpToHeight: 0, + pendingSyncOfHeights: [], + }); + const pendingThroughHeight = 4_000; + const maximumDurationMs = 1_000; + + const startedAt = Date.now(); + for ( + let nextHeight = state.getPendingSyncHeight() + 1; + nextHeight <= pendingThroughHeight; + nextHeight = state.getPendingSyncHeight() + 1 + ) { + state.markHeightAsPendingSync(nextHeight); + } + const durationMs = Date.now() - startedAt; + + t.deepEqual(state.getPendingSyncHeight(), pendingThroughHeight); + t.true( + durationMs < maximumDurationMs, + `Expected marking contiguous pending heights to complete in less than ${maximumDurationMs}ms; took ${durationMs}ms.` + ); +}); From 535e41b187dc12ef947895ce3ad572ffbf35325f Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 13 May 2026 10:48:13 +0100 Subject: [PATCH 30/63] fix(agent): make header catch-up sync state linear --- src/components/sync-state.spec.ts | 2 + src/components/sync-state.ts | 66 +++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/components/sync-state.spec.ts b/src/components/sync-state.spec.ts index 634bb7b1..4f5fb3cc 100644 --- a/src/components/sync-state.spec.ts +++ b/src/components/sync-state.spec.ts @@ -69,7 +69,9 @@ test('SyncState scales when marking many contiguous heights as pending', (t) => const maximumDurationMs = 1_000; const startedAt = Date.now(); + // eslint-disable-next-line functional/no-loop-statement for ( + // eslint-disable-next-line functional/no-let let nextHeight = state.getPendingSyncHeight() + 1; nextHeight <= pendingThroughHeight; nextHeight = state.getPendingSyncHeight() + 1 diff --git a/src/components/sync-state.ts b/src/components/sync-state.ts index ed15b605..58f6e5c8 100644 --- a/src/components/sync-state.ts +++ b/src/components/sync-state.ts @@ -48,10 +48,21 @@ export class SyncState { */ latestSyncedBlockTime: Date | 'caught-up' | undefined; + private pendingSyncHeight: number; + + private pendingSyncHeights: Set; + + private additionalSyncedHeightSet: Set; + constructor(initialState: InitialSyncState) { this.fullySyncedUpToHeight = initialState.fullySyncedUpToHeight; this.pendingSyncOfHeights = initialState.pendingSyncOfHeights.slice(); this.additionalSyncedHeights = initialState.additionalSyncedHeights.slice(); + this.pendingSyncHeights = new Set(this.pendingSyncOfHeights); + this.additionalSyncedHeightSet = new Set(this.additionalSyncedHeights); + this.pendingSyncHeight = this.computePendingSyncHeightFrom( + this.fullySyncedUpToHeight + ); this.latestSyncedBlockTime = initialState.fullySyncedUpToHeight > 0 ? 'caught-up' : undefined; } @@ -69,28 +80,35 @@ export class SyncState { if ( this.fullySyncedUpToHeight < height && - !this.additionalSyncedHeights.includes(height) + !this.additionalSyncedHeightSet.has(height) ) { this.additionalSyncedHeights.push(height); - removeValueIfPresent(this.pendingSyncOfHeights, height); + this.additionalSyncedHeightSet.add(height); + if (this.pendingSyncHeights.delete(height)) { + removeValueIfPresent(this.pendingSyncOfHeights, height); + } } // eslint-disable-next-line functional/no-let let nextHeight = this.fullySyncedUpToHeight + 1; // eslint-disable-next-line functional/no-loop-statement - while (removeValueIfPresent(this.additionalSyncedHeights, nextHeight)) { + while (this.additionalSyncedHeightSet.delete(nextHeight)) { + removeValueIfPresent(this.additionalSyncedHeights, nextHeight); this.fullySyncedUpToHeight = nextHeight; nextHeight += 1; } + this.updatePendingSyncHeight(); } markHeightAsPendingSync(height: number) { if ( this.fullySyncedUpToHeight < height && - !this.pendingSyncOfHeights.includes(height) && - !this.additionalSyncedHeights.includes(height) + !this.pendingSyncHeights.has(height) && + !this.additionalSyncedHeightSet.has(height) ) { this.pendingSyncOfHeights.push(height); + this.pendingSyncHeights.add(height); + this.updatePendingSyncHeight(); } } @@ -113,6 +131,9 @@ export class SyncState { this.additionalSyncedHeights = this.additionalSyncedHeights.filter( (completed) => completed < height ); + this.pendingSyncHeights = new Set(this.pendingSyncOfHeights); + this.additionalSyncedHeightSet = new Set(this.additionalSyncedHeights); + this.resetPendingSyncHeight(); } /** @@ -120,16 +141,35 @@ export class SyncState { * useful for prioritizing syncing. */ getPendingSyncHeight() { - const handledHeights = [ - ...this.pendingSyncOfHeights, - ...this.additionalSyncedHeights, - ]; + return this.pendingSyncHeight; + } + + private isHeightHandled(height: number) { + return ( + this.pendingSyncHeights.has(height) || + this.additionalSyncedHeightSet.has(height) + ); + } + + private computePendingSyncHeightFrom(height: number) { // eslint-disable-next-line functional/no-let - let firstUnhandledHeight = this.fullySyncedUpToHeight + 1; + let pendingSyncHeight = height; // eslint-disable-next-line functional/no-loop-statement - while (handledHeights.includes(firstUnhandledHeight)) { - firstUnhandledHeight += 1; + while (this.isHeightHandled(pendingSyncHeight + 1)) { + pendingSyncHeight += 1; } - return firstUnhandledHeight - 1; + return pendingSyncHeight; + } + + private updatePendingSyncHeight() { + this.pendingSyncHeight = this.computePendingSyncHeightFrom( + Math.max(this.pendingSyncHeight, this.fullySyncedUpToHeight) + ); + } + + private resetPendingSyncHeight() { + this.pendingSyncHeight = this.computePendingSyncHeightFrom( + this.fullySyncedUpToHeight + ); } } From a270c0c811ef38c10cfe10862fd8dc485f5d71e2 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Fri, 12 Jun 2026 15:11:13 +0100 Subject: [PATCH 31/63] test(agent): new block saved after reorg (#77) --- src/e2e/e2e.spec.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 8c8034b5..e0a9a5b1 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -1980,6 +1980,34 @@ test.serial('[e2e] handles re-org of a single block', async (t) => { t.pass(); }); +test.serial( + '[e2e] new block saved after reorg', + async (t) => { + const acceptedBlocks = ( + await client.query<{ + hash: string; + nodeName: string; + }>( + /* sql */ ` + SELECT node.name AS "nodeName", encode(block.hash, 'hex') AS hash + FROM node_block + INNER JOIN node + ON node.internal_id = node_block.node_internal_id + INNER JOIN block + ON block.internal_id = node_block.block_internal_id + WHERE node.name = 'node3' + AND block.height = $1 + ORDER BY block.hash; + `, + [splitHeight + 1] + ) + ).rows; + t.deepEqual(acceptedBlocks, [ + { hash: tipA[0]!.header.hash, nodeName: 'node3' }, + ]); + } +); + test.serial('[e2e] handles reversal of single-block re-org', async (t) => { const tipStartIndex = 2; const tipEnd = 6; From 31b789139804453c25bd16d8e5813196ba89948c Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Fri, 12 Jun 2026 16:10:53 +0100 Subject: [PATCH 32/63] fix(agent): save replacement block on reorg (fixes #77) --- src/agent.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/agent.ts b/src/agent.ts index 2dbe80ed..f08c12ed 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -1753,6 +1753,7 @@ export class Agent { nodeAcceptances, transactionCache: this.transactionCache, }); + this.blockDb?.add(block.hash); const completionTime = Date.now(); const durationMs = completionTime - startTime; @@ -1841,12 +1842,15 @@ export class Agent { firstHeight: number, nodeName: string ) { - removeStaleBlocksForNode(this.nodes[nodeName]!.internalId!, staleChain) + const node = this.nodes[nodeName]!; + node.syncState?.blockReorganizationAtHeight(firstHeight); + removeStaleBlocksForNode(node.internalId!, staleChain) .then(() => { this.logger.info( staleChain, `${nodeName}: re-organization detected beginning at height: ${firstHeight}. The following stale blocks were removed:` ); + this.scheduleBlockBufferFill(); }) .catch((err) => { this.logger.error(err); From 253207f4d88157ad3b33d03929c3dd6547ef38b0 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 23 Jul 2026 17:00:37 +0100 Subject: [PATCH 33/63] fix(chart): use Recreate strategy for agent deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default RollingUpdate strategy runs old and new agent pods concurrently until the new pod passes readiness (minutes of node init). The agent is a singleton writer — inserts are idempotent but reorg handling assumes exclusive access to node_block, so an overlapping agent can transiently resurrect stale-chain acceptance rows. Recreate guarantees at most one agent at the cost of a brief indexing gap per rollout. Co-Authored-By: Claude Fable 5 --- charts/chaingraph/templates/agent.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/chaingraph/templates/agent.yaml b/charts/chaingraph/templates/agent.yaml index 2388432b..b53ec8ca 100644 --- a/charts/chaingraph/templates/agent.yaml +++ b/charts/chaingraph/templates/agent.yaml @@ -8,6 +8,14 @@ metadata: {{- include "chaingraph.labels" . | nindent 4 }} spec: replicas: 1 + # The agent is a singleton writer: never run two concurrently. Recreate + # terminates the old pod before starting its replacement, at the cost of a + # brief indexing gap (the agent catches up on restart). + strategy: + type: Recreate + # Explicit null clears the server-defaulted rollingUpdate block left by + # previous RollingUpdate deployments (Helm three-way merge). + rollingUpdate: null selector: matchLabels: app: chaingraph-agent From b9a0c3b6445856da22d2b9093695fced1ea922d9 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 23 Jul 2026 17:03:20 +0100 Subject: [PATCH 34/63] feat(hasura): upgrade hasura to v2.49.5 3.5 years of CVE and performance fixes over v2.16.1, staying on the v2 line. First startup against an existing database performs a one-way metadata catalog upgrade (47 -> 48); v2.16.1 will not start against the upgraded catalog, so snapshot hdb_catalog before deploying. Rehearsed against a schema-only copy of the production database (including its catalog-47 hdb_catalog): migration completes, metadata is consistent, and existing query shapes execute unchanged. Co-Authored-By: Claude Fable 5 --- images/hasura/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/images/hasura/Dockerfile b/images/hasura/Dockerfile index f59c2cd3..d345f0ec 100644 --- a/images/hasura/Dockerfile +++ b/images/hasura/Dockerfile @@ -1,5 +1,10 @@ -FROM hasura/graphql-engine:v2.16.1.cli-migrations-v3 +# Multi-arch tag; digest of the manifest list as of 2026-07-20: +# sha256:da573f6420bfbc330023dd4b816c1e6fcb50ab6a00fd3e1d8c70088532d5fa7f +# NOTE: first startup against an existing database performs a ONE-WAY +# metadata catalog upgrade (47 -> current). Snapshot/backup hdb_catalog +# before deploying to an instance you may want to roll back. +FROM hasura/graphql-engine:v2.49.5.cli-migrations-v3 COPY hasura-data /hasura-data From aba2803ab83ba3a47e163ab4fa048ac0aaf25ae6 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 23 Jul 2026 17:19:19 +0100 Subject: [PATCH 35/63] fix(agent): speed up block hash restore and correct its startup warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore hashes with encode(hash, 'hex') in Postgres instead of materializing ~1.5M Buffers and hex-encoding each on the agent event loop, which dominated startup on large databases. Log the restore count and duration on completion. Replace the misleading "database configuration or connectivity problem" warning — which fired after just 6 seconds of a legitimately slow restore — with an accurate "still restoring" message gated to 15s. e2e: verify the SQL-side encoding matches the previous client-side conversion exactly. Co-Authored-By: Claude Fable 5 --- src/agent.ts | 57 ++++++++++++++++++++++++++++++++------------- src/db.ts | 14 ++++++----- src/e2e/e2e.spec.ts | 38 ++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 22 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index f08c12ed..602802f8 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -361,6 +361,14 @@ export class Agent { blockDbRestored = false; + /** + * Restoring the block hash set (`SELECT encode("hash", 'hex') FROM block`) + * legitimately takes tens of seconds on large databases, so don't warn about + * it before this many seconds have elapsed. + */ + // eslint-disable-next-line @typescript-eslint/no-magic-numbers + blockDbRestoreWarnDelaySeconds = 15; + successfullyInitialized = false; /** @@ -426,10 +434,19 @@ export class Agent { targetSize: blockBufferTargetSizeMb * MB, }); + const blockDbRestoreStart = Date.now(); // eslint-disable-next-line @typescript-eslint/no-floating-promises getAllKnownBlockHashes().then((hashes) => { this.blockDb = new Set(hashes); this.blockDbRestored = true; + const millisecondsPerSecond = 1000; + const restoreSeconds = ( + (Date.now() - blockDbRestoreStart) / + millisecondsPerSecond + ).toFixed(1); + this.logger.info( + `Restored ${hashes.length} known block hashes from the database in ${restoreSeconds} seconds.` + ); }); this.logger.info( @@ -765,6 +782,29 @@ export class Agent { * issue requires that Chaingraph be restarted. */ + warnIfInitializationDelayed(uninitializedNodes: string[]) { + const second = 1000; + const currentDelay = Date.now() - this.startTime.getTime(); + const currentDelaySecondsRounded = Math.round(currentDelay / second); + if (this.warnAfterDelaySecond >= currentDelaySecondsRounded) { + return; + } + this.warnAfterDelaySecond = currentDelaySecondsRounded; + if (this.blockDbRestored) { + this.logger.warn( + `The following nodes are taking longer than ${currentDelaySecondsRounded} seconds to initialize and may be either misconfigured or unresponsive: ${uninitializedNodes.join( + ',' + )}` + ); + } else if ( + currentDelaySecondsRounded >= this.blockDbRestoreWarnDelaySeconds + ) { + this.logger.warn( + `Still restoring known block hashes from the database (${currentDelaySecondsRounded} seconds elapsed). This can take a minute or more on large databases; if it never completes, check database configuration and connectivity.` + ); + } + } + checkInitialization() { const uninitializedNodes = Object.entries(this.nodes).reduce( (nodes, [id, node]) => @@ -773,22 +813,7 @@ export class Agent { ); if (uninitializedNodes.length > 0 || !this.blockDbRestored) { const second = 1000; - const currentDelay = Date.now() - this.startTime.getTime(); - const currentDelaySecondsRounded = Math.round(currentDelay / second); - if (this.warnAfterDelaySecond < currentDelaySecondsRounded) { - this.warnAfterDelaySecond = currentDelaySecondsRounded; - if (this.blockDbRestored) { - this.logger.warn( - `The following nodes are taking longer than ${currentDelaySecondsRounded} seconds to initialize and may be either misconfigured or unresponsive: ${uninitializedNodes.join( - ',' - )}` - ); - } else { - this.logger.warn( - `Could not restore block hashes from the database after ${currentDelaySecondsRounded} seconds. There may be a database configuration or connectivity problem.` - ); - } - } + this.warnIfInitializationDelayed(uninitializedNodes); setTimeout(() => { this.scheduleBlockBufferFill(); }, second); diff --git a/src/db.ts b/src/db.ts index fa4465e5..c5b28be1 100644 --- a/src/db.ts +++ b/src/db.ts @@ -84,14 +84,16 @@ export const blockArrayToHashChain = ( */ export const getAllKnownBlockHashes = async () => { const client = await pool.connect(); - const allKnownBlockHashes = await client.query<{ hash: Buffer }>( - 'SELECT "hash" from "block";' + /* + * Hex-encoding in Postgres avoids materializing millions of `Buffer`s and + * converting each to hex on the (single-threaded) agent event loop, which + * dominated startup time on large databases. + */ + const allKnownBlockHashes = await client.query<{ hash: string }>( + `SELECT encode("hash", 'hex') AS "hash" from "block";` ); client.release(); - const hashes = allKnownBlockHashes.rows.map(({ hash }) => - hash.toString('hex') - ); - return hashes; + return allKnownBlockHashes.rows.map(({ hash }) => hash); }; export interface IncompleteBlock { diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index e0a9a5b1..7d4d7588 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -844,6 +844,44 @@ test.serial('[e2e] creates expected indexes after initial sync', async (t) => { t.pass(); }); +test.serial( + '[e2e] getAllKnownBlockHashes returns hex hashes for every known block', + async (t) => { + const originalPostgresConnectionString = + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + postgresE2eConnectionStringTestDb; + const { getAllKnownBlockHashes } = await import('../db.js'); + // eslint-disable-next-line functional/no-try-statement + try { + const hashes = await getAllKnownBlockHashes(); + /* + * Convert client-side (the previous implementation) to verify the + * SQL-side `encode(...)` used by `getAllKnownBlockHashes` matches it. + */ + const expected = ( + await client.query<{ hash: Buffer }>(/* sql */ ` + SELECT hash FROM block ORDER BY hash; + `) + ).rows.map((row) => row.hash.toString('hex')); + t.true(expected.length > 0); + t.deepEqual( + [...hashes].sort((a, b) => (a < b ? -1 : Number(a > b))), + expected + ); + const hexPattern = /^[0-9a-f]{64}$/u; + t.true(hashes.every((hash) => hexPattern.test(hash))); + } finally { + if (originalPostgresConnectionString === undefined) { + delete process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + } else { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + originalPostgresConnectionString; + } + } + } +); + test.serial( '[e2e] records node validation after concurrent transaction insert conflict', async (t) => { From 35883a83ced7725d8ea1292c52ad430e62b735d6 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 23 Jul 2026 17:28:17 +0100 Subject: [PATCH 36/63] fix(agent): phase-aware node initialization warnings with restore timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "misconfigured or unresponsive" warning fired for nodes whose P2P connection was ready but whose chain state was still being restored from the database — legitimate work proportional to chain length (~9s for a 960k-block chain sharing the pool with the block hash restore). Split the warning: nodes without a ready P2P connection keep the misconfigured/unresponsive text; connected nodes get an accurate "still restoring chain state" message. Log each node's registration + chain restore duration on completion, and let the block hash restore warning use the same 5-second base patience as node warnings. Co-Authored-By: Claude Fable 5 --- src/agent.ts | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index 602802f8..be2b08bd 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -361,14 +361,6 @@ export class Agent { blockDbRestored = false; - /** - * Restoring the block hash set (`SELECT encode("hash", 'hex') FROM block`) - * legitimately takes tens of seconds on large databases, so don't warn about - * it before this many seconds have elapsed. - */ - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - blockDbRestoreWarnDelaySeconds = 15; - successfullyInitialized = false; /** @@ -554,6 +546,7 @@ export class Agent { */ peer.sendMessage(new peer.messages.SendHeaders()); + const nodeRegistrationStart = Date.now(); // eslint-disable-next-line @typescript-eslint/no-floating-promises registerTrustedNodeWithDb({ latestConnectionBeganAt: new Date(), @@ -576,6 +569,14 @@ export class Agent { : syncedHeaderHashChain ); this.nodes[node.name]!.syncState = new SyncState(initialSyncState); + const millisecondsPerSecond = 1000; + const chainRestoreSeconds = ( + (Date.now() - nodeRegistrationStart) / + millisecondsPerSecond + ).toFixed(1); + this.logger.info( + `${node.name}: chain state registered and restored from the database in ${chainRestoreSeconds} seconds.` + ); const genesisBlockHeaderFromDb = this.blockTree.getBlockHeaderHash( node.name, 0 @@ -790,17 +791,35 @@ export class Agent { return; } this.warnAfterDelaySecond = currentDelaySecondsRounded; - if (this.blockDbRestored) { + if (!this.blockDbRestored) { this.logger.warn( - `The following nodes are taking longer than ${currentDelaySecondsRounded} seconds to initialize and may be either misconfigured or unresponsive: ${uninitializedNodes.join( + `Still restoring known block hashes from the database (${currentDelaySecondsRounded} seconds elapsed). This can take a minute or more on large databases; if it never completes, check database configuration and connectivity.` + ); + return; + } + /* + * A node whose P2P connection is `ready` is not unresponsive – it is + * waiting on its chain state to be registered and restored from the + * database, which takes time proportional to the node's chain length. + */ + const unconnectedNodes = uninitializedNodes.filter( + (id) => this.nodes[id]!.peer.status !== 'ready' + ); + const restoringChainNodes = uninitializedNodes.filter( + (id) => this.nodes[id]!.peer.status === 'ready' + ); + if (unconnectedNodes.length > 0) { + this.logger.warn( + `The following nodes are taking longer than ${currentDelaySecondsRounded} seconds to connect and may be either misconfigured or unresponsive: ${unconnectedNodes.join( ',' )}` ); - } else if ( - currentDelaySecondsRounded >= this.blockDbRestoreWarnDelaySeconds - ) { + } + if (restoringChainNodes.length > 0) { this.logger.warn( - `Still restoring known block hashes from the database (${currentDelaySecondsRounded} seconds elapsed). This can take a minute or more on large databases; if it never completes, check database configuration and connectivity.` + `Still restoring chain state from the database for: ${restoringChainNodes.join( + ',' + )} (${currentDelaySecondsRounded} seconds elapsed). Restore time is proportional to each node's chain length.` ); } } From c09fb714fd55bf234be428c317470c9ad2c4b727 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 23 Jul 2026 17:35:50 +0100 Subject: [PATCH 37/63] build(agent): upgrade Node 18 to 24 LTS in agent image and engines Node 18 has been EOL since April 2025. The e2e suite already runs under Node 24 locally (89 tests passing), and the agent boots cleanly in the node:24-alpine image. engines raised to >=22. Co-Authored-By: Claude Fable 5 --- images/agent/Dockerfile | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/images/agent/Dockerfile b/images/agent/Dockerfile index f0a0d499..67a845bf 100644 --- a/images/agent/Dockerfile +++ b/images/agent/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-alpine AS build-stage +FROM node:24-alpine AS build-stage WORKDIR /chaingraph COPY package.json yarn.lock .yarnrc.yml ./ COPY .yarn .yarn @@ -10,7 +10,7 @@ RUN yarn build # Use Yarn plugin-production-install to copy only production dependencies (https://gitlab.com/Larry1123/yarn-contrib/-/tree/master/packages/plugin-production-install) RUN yarn prod-install ./production-dependencies -FROM node:18-alpine AS production +FROM node:24-alpine AS production WORKDIR /chaingraph COPY --from=build-stage /chaingraph/production-dependencies ./ COPY --from=build-stage /chaingraph/defaults.env ./ diff --git a/package.json b/package.json index c93c11cb..8ae223a5 100644 --- a/package.json +++ b/package.json @@ -142,7 +142,7 @@ "postinstall": "touch .env" }, "engines": { - "node": ">=10" + "node": ">=22" }, "dependencies": { "@bitauth/libauth": "^2.0.0-alpha.8", From c5cd4da12d135d072eca276e2c56af8d5b8b6234 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 23 Jul 2026 17:40:07 +0100 Subject: [PATCH 38/63] build: upgrade pg 8.8 to 8.22 and @types/pg to 8.20 Same-major driver bump carrying 3.5 years of fixes; e2e suite (89 tests, real sync against Postgres 18) passes unchanged. Co-Authored-By: Claude Fable 5 --- .yarn | 2 +- package.json | 4 +-- yarn.lock | 88 +++++++++++++++++++++++++++------------------------- 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/.yarn b/.yarn index a7a702dd..d33cb1f7 160000 --- a/.yarn +++ b/.yarn @@ -1 +1 @@ -Subproject commit a7a702dde60fc7eb627f4e7a655d6063590a4e69 +Subproject commit d33cb1f7cfb2386378c6a9c93564e79153417fcb diff --git a/package.json b/package.json index 8ae223a5..9c81f48f 100644 --- a/package.json +++ b/package.json @@ -149,7 +149,7 @@ "@chaingraph/bitcore-p2p-cash": "^9.0.5", "dotenv": "^16.0.3", "lru-cache": "^7.14.1", - "pg": "^8.8.0", + "pg": "^8.22.0", "pino": "^8.8.0" }, "devDependencies": { @@ -157,7 +157,7 @@ "@istanbuljs/nyc-config-typescript": "^1.0.2", "@types/dotenv": "^8.2.0", "@types/lru-cache": "^7.10.10", - "@types/pg": "^8.6.6", + "@types/pg": "^8.20.0", "@typescript-eslint/eslint-plugin": "^5.48.1", "@typescript-eslint/parser": "^5.48.1", "ava": "^5.1.0", diff --git a/yarn.lock b/yarn.lock index 2f10a41b..7eed91bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1362,14 +1362,14 @@ __metadata: languageName: node linkType: hard -"@types/pg@npm:^8.6.6": - version: 8.6.6 - resolution: "@types/pg@npm:8.6.6" +"@types/pg@npm:^8.20.0": + version: 8.20.0 + resolution: "@types/pg@npm:8.20.0" dependencies: "@types/node": "*" pg-protocol: "*" pg-types: ^2.2.0 - checksum: ac145553a8ad2f357feacad1bceaf5d6ce904eb9d66233b84c469a2b4fa3738d4ebdf29b7ea45387be2d07f915fd873a229f90a2f766d7c377afa7c41fbcf8d1 + checksum: e3610a00e38d52a0b9069bda48a9a5746d198f0aaa72710898fe6c277777ca3ef5190725afa61f53adb4ddd624e48267133b691f4ae7076dbef5d83aa31c2cf0 languageName: node linkType: hard @@ -2634,13 +2634,6 @@ __metadata: languageName: node linkType: hard -"buffer-writer@npm:2.0.0": - version: 2.0.0 - resolution: "buffer-writer@npm:2.0.0" - checksum: 11736b48bb75106c52ca8ec9f025e7c1b3b25ce31875f469d7210eabd5c576c329e34f6b805d4a8d605ff3f0db1e16342328802c4c963e9c826b0e43a4e631c2 - languageName: node - linkType: hard - "buffer-xor@npm:^1.0.3": version: 1.0.3 resolution: "buffer-xor@npm:1.0.3" @@ -2892,7 +2885,7 @@ __metadata: "@istanbuljs/nyc-config-typescript": ^1.0.2 "@types/dotenv": ^8.2.0 "@types/lru-cache": ^7.10.10 - "@types/pg": ^8.6.6 + "@types/pg": ^8.20.0 "@typescript-eslint/eslint-plugin": ^5.48.1 "@typescript-eslint/parser": ^5.48.1 ava: ^5.1.0 @@ -2913,7 +2906,7 @@ __metadata: npm-run-all: ^4.1.5 nyc: ^15.1.0 open-cli: ^7.1.0 - pg: ^8.8.0 + pg: ^8.22.0 pino: ^8.8.0 pino-pretty: ^9.1.1 prando: ^6.0.1 @@ -9541,13 +9534,6 @@ __metadata: languageName: node linkType: hard -"packet-reader@npm:1.0.0": - version: 1.0.0 - resolution: "packet-reader@npm:1.0.0" - checksum: 0b7516f0cbf3e322aad591bed29ba544220088c53943145c0d9121a6f59182ad811f7fd6785a8979a34356aca69d97653689029964c5998dc02645633d88ffd7 - languageName: node - linkType: hard - "pako@npm:^1.0.3, pako@npm:~1.0.5": version: 1.0.11 resolution: "pako@npm:1.0.11" @@ -9744,10 +9730,17 @@ __metadata: languageName: node linkType: hard -"pg-connection-string@npm:^2.5.0": - version: 2.5.0 - resolution: "pg-connection-string@npm:2.5.0" - checksum: a6f3a068f7c9416a5b33a326811caf0dfaaee045c225b7c628b4c9b4e9a2b25bdd12a21e4c48940e1000ea223a4e608ca122d2ff3dd08c8b1db0fc9f5705133a +"pg-cloudflare@npm:^1.4.0": + version: 1.4.0 + resolution: "pg-cloudflare@npm:1.4.0" + checksum: cddea0ab52b4c4a3c842a2304ccc10ef61e06fd00ba633f4c2c04b31e9352adee8ee0a7b1fdc8e8738527d3e39534f75903f467a0b1a6af6cc49fb5abe90481c + languageName: node + linkType: hard + +"pg-connection-string@npm:^2.14.0": + version: 2.14.0 + resolution: "pg-connection-string@npm:2.14.0" + checksum: 10e798bedffa7da43eaf6879c134c2f6e6344185e1ee3856c92444848d879ba7f0cce4981c8e3d58baf100d2da845d11b88cb9a788552d0a65bdeae71a338dce languageName: node linkType: hard @@ -9758,23 +9751,30 @@ __metadata: languageName: node linkType: hard -"pg-pool@npm:^3.5.2": - version: 3.5.2 - resolution: "pg-pool@npm:3.5.2" +"pg-pool@npm:^3.14.0": + version: 3.14.0 + resolution: "pg-pool@npm:3.14.0" peerDependencies: pg: ">=8.0" - checksum: a5d029200257671f0c17ca54b9ab6213e2060e64e5cc792b78edd50ab471a26cd364890d05d546d9296949e079e943821cf2ceb4d488f4e6a6789fb781a628bf + checksum: cf077bace1c7a55011322c71973cf22fb152bf71312c15484dc75818ace77f7759d96069fb0125163f6fa56c5470a0ea70734c2f6218d398018e4ecf835bf40e languageName: node linkType: hard -"pg-protocol@npm:*, pg-protocol@npm:^1.5.0": +"pg-protocol@npm:*": version: 1.5.0 resolution: "pg-protocol@npm:1.5.0" checksum: b839d12cafe942ef9cbc5b13c174eb2356804fb4fe8ead8279f46a36be90722d19a91409955beb8a3d5301639c44854e49749de4aef02dc361fee3e2a61fb1e4 languageName: node linkType: hard -"pg-types@npm:^2.1.0, pg-types@npm:^2.2.0": +"pg-protocol@npm:^1.15.0": + version: 1.15.0 + resolution: "pg-protocol@npm:1.15.0" + checksum: eef533572ab655dada3a897646194d81b4901bd1729b134c2d5187cc617f71d1a77465efb6ab825486e35cbb8a0311f7ad3893f60e16c9707c0e31f05373a7c3 + languageName: node + linkType: hard + +"pg-types@npm:2.2.0, pg-types@npm:^2.2.0": version: 2.2.0 resolution: "pg-types@npm:2.2.0" dependencies: @@ -9787,27 +9787,29 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.8.0": - version: 8.8.0 - resolution: "pg@npm:8.8.0" - dependencies: - buffer-writer: 2.0.0 - packet-reader: 1.0.0 - pg-connection-string: ^2.5.0 - pg-pool: ^3.5.2 - pg-protocol: ^1.5.0 - pg-types: ^2.1.0 - pgpass: 1.x +"pg@npm:^8.22.0": + version: 8.22.0 + resolution: "pg@npm:8.22.0" + dependencies: + pg-cloudflare: ^1.4.0 + pg-connection-string: ^2.14.0 + pg-pool: ^3.14.0 + pg-protocol: ^1.15.0 + pg-types: 2.2.0 + pgpass: 1.0.5 peerDependencies: pg-native: ">=3.0.1" + dependenciesMeta: + pg-cloudflare: + optional: true peerDependenciesMeta: pg-native: optional: true - checksum: fa30a85814dd7238b582c3bc6c0b9e2b0ae38dd0a6bb485ef480e64bb5ce589de6cb873ce4d3cd10c37a3e0a1e1281ba75dc7d80b1a68bae91999cd5b70d398b + checksum: 68695b3427476bc5a8838fddb4ab56f091319d74d45ee20c3e9fc1414c848bc9535ef4b53977c4407c511296a1848a543ba7824bd4652fd9c146e0e04bc4935b languageName: node linkType: hard -"pgpass@npm:1.x": +"pgpass@npm:1.0.5": version: 1.0.5 resolution: "pgpass@npm:1.0.5" dependencies: From b19783b9c8d602f8137be619462f56ba12de2788 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 23 Jul 2026 19:09:42 +0100 Subject: [PATCH 39/63] fix(chart): tighten agent readiness probe, keep liveness coarse /health-check answers ~1s after start, but both probes waited 30s before the first check, so pods showed NotReady for 30+ seconds. Readiness now 5s delay / 10s period / 5s timeout; liveness stays coarse so event-loop stalls during catch-up don't get pods killed. Co-Authored-By: Claude Fable 5 --- charts/chaingraph/templates/agent.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/charts/chaingraph/templates/agent.yaml b/charts/chaingraph/templates/agent.yaml index b53ec8ca..86eaaa24 100644 --- a/charts/chaingraph/templates/agent.yaml +++ b/charts/chaingraph/templates/agent.yaml @@ -78,13 +78,17 @@ spec: timeoutSeconds: 15 successThreshold: 1 failureThreshold: 3 + # Readiness is tight so rollouts report Ready seconds after the + # internal API is up. Liveness (above) stays coarse on purpose: a + # CPU-bound catch-up can block the event loop (and /health-check), + # and a twitchy liveness probe would kill the agent mid-recovery. readinessProbe: httpGet: path: /health-check port: {{ .Values.agent.port }} - initialDelaySeconds: 30 - periodSeconds: 60 - timeoutSeconds: 15 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 resources: {} From d7a41b750449c043e3e640a4da278d1965e7701d Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 4 Oct 2026 14:18:30 +0100 Subject: [PATCH 40/63] test(sql): search_output must match locking bytecode of any length exactly search_output compares each input, unchanged, with the first 25 bytes of every locking bytecode, so inputs longer than 25 bytes (all P2SH32 outputs) never match, and 25-byte inputs also match longer scripts sharing that prefix. Add a failing e2e test over synthetic outputs (inserted in a rolled-back transaction). Refs #78 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/e2e/e2e.spec.ts | 84 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 7d4d7588..41a794e4 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -3142,3 +3142,87 @@ test('[e2e] [postgres] encode_compact_uint', async (t) => { t.deepEqual(await query(BigInt('9223372036854775807')), 'ffffffffffffffff7f'); /* eslint-enable @typescript-eslint/no-magic-numbers */ }); + +/* cspell: disable */ +const searchFixtureTxHash = + 'f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0'; +const searchFixtureOutputs = { + /** P2PKH whose hash contains 0x5c (the LIKE escape character) */ + backslash: '76a9144444444444444444445c4444444444444444444488ac', + /** P2PKH that only a wildcard reading of 0x25 or 0x5f would match */ + other: '76a914997777777777777777777777777777777777777788ac', + /** 25-byte P2PKH */ + p2pkh: '76a914111111111111111111111111111111111111111188ac', + /** a longer script sharing the P2PKH's first 25 bytes */ + p2pkhPrefixed: '76a914111111111111111111111111111111111111111188acab', + /** 35-byte P2SH32 */ + p2sh32: + 'aa20222222222222222222222222222222222222222222222222222222222222222287', + /** shares the P2SH32's first 25 bytes, differs afterwards */ + p2sh32SamePrefix: + 'aa20222222222222222222222222222222222222222222333333333333333333333387', + /** P2PKH whose hash starts with 0x25 (LIKE "%") */ + percent: '76a914255555555555555555555555555555555555555588ac', + /** P2PKH whose hash starts with 0x5f (LIKE "_") */ + underscore: '76a9145f6666666666666666666666666666666666666688ac', +}; +/* cspell: enable */ + +/** + * Insert synthetic outputs in a database transaction that is always rolled + * back, run `query` restricted to the fixture transaction, and return the + * names of the matched fixture outputs. + */ +const searchFixtureMatches = async (query: string, parameter: unknown) => { + const names = Object.keys(searchFixtureOutputs); + await client.query('BEGIN;'); + const insertAndQuery = async () => { + await client.query( + /* sql */ `INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) VALUES ($1, 2, 0, 0, false);`, + [hexToBin(searchFixtureTxHash)] + ); + await Object.values(searchFixtureOutputs).reduce>( + async (chain, bytecode, index) => + chain.then(async () => + client.query( + /* sql */ `INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) VALUES ($1, $2, 1000, $3);`, + [hexToBin(searchFixtureTxHash), index, hexToBin(bytecode)] + ) + ), + Promise.resolve(undefined) + ); + const result = await client.query<{ outputIndex: string }>( + /* sql */ `SELECT output_index AS "outputIndex" FROM ${query} AS o WHERE o.transaction_hash = $2 ORDER BY output_index;`, + [parameter, hexToBin(searchFixtureTxHash)] + ); + return result.rows.map((row) => names[Number(row.outputIndex)]); + }; + const rollback = async () => client.query('ROLLBACK;'); + return insertAndQuery().then( + async (matches) => rollback().then(() => matches), + // eslint-disable-next-line functional/no-promise-reject -- roll back, then propagate the original failure + async (error: unknown) => rollback().then(async () => Promise.reject(error)) + ); +}; + +test.serial( + '[e2e] [sql] search_output: exact matches for locking bytecode of any length', + async (t) => { + const search = async (scripts: string[]) => + searchFixtureMatches('search_output($1::text[])', scripts); + t.deepEqual(await search([searchFixtureOutputs.p2sh32]), ['p2sh32']); + t.deepEqual(await search([searchFixtureOutputs.p2pkh]), ['p2pkh']); + t.deepEqual(await search([searchFixtureOutputs.p2pkhPrefixed]), [ + 'p2pkhPrefixed', + ]); + t.deepEqual( + await search([ + searchFixtureOutputs.p2sh32SamePrefix, + searchFixtureOutputs.p2pkh, + searchFixtureOutputs.p2pkh, + ]), + ['p2pkh', 'p2sh32SamePrefix'] + ); + t.deepEqual(await search([]), []); + } +); From 702ef283217b39a5ad19364875a96e24198962e9 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 4 Oct 2026 14:18:30 +0100 Subject: [PATCH 41/63] fix(sql): make search_output exact for locking bytecode of any length Truncate each input to 25 bytes for the indexed comparison against output_search_index, then recheck full equality, so P2SH32 and other scripts longer than 25 bytes are found and 25-byte inputs no longer match longer scripts sharing that prefix. Rewrite as an inlinable SQL function so Hasura's limit and filters apply inside the query. Fixes #78 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../down.sql | 17 ++++++++++++++++ .../up.sql | 20 +++++++++++++++++++ src/e2e/e2e.spec.ts | 1 + 3 files changed, 38 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/down.sql b/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/down.sql new file mode 100644 index 00000000..53c12d2d --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/down.sql @@ -0,0 +1,17 @@ +-- Restore the original search_output definition from 1616195337538_init. +CREATE OR REPLACE FUNCTION search_output (locking_bytecode_hex text[]) + RETURNS SETOF output + LANGUAGE plpgsql STABLE +AS $$ +DECLARE + decoded bytea[]; +BEGIN + SELECT array_agg(decode(hex, 'hex')) into decoded FROM unnest(locking_bytecode_hex) as hex; + RETURN QUERY (SELECT * FROM output + -- use output_locking_bytecode_prefix_index + WHERE substring(locking_bytecode from 0 for 26) + = ANY(decoded) + ORDER BY locking_bytecode ASC); +END; +$$; +COMMENT ON FUNCTION search_output (text[]) IS 'Return a list of outputs which match the provided locking bytecode hex (up to 25 bytes, supporting both P2PKH and P2SH outputs).'; diff --git a/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/up.sql b/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/up.sql new file mode 100644 index 00000000..97358cc0 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/up.sql @@ -0,0 +1,20 @@ +-- search_output previously compared each input, unchanged, with the 25-byte +-- prefix stored by output_search_index, so inputs longer than 25 bytes (e.g. +-- P2SH32) never matched and 25-byte inputs also matched longer scripts sharing +-- that prefix. Truncate each input to 25 bytes for the indexed comparison, then +-- recheck full equality. As a single-statement SQL function, PostgreSQL can +-- inline it, so Hasura's where/limit/order_by apply inside the query. +-- Note: this method expects an index: CREATE INDEX output_search_index ON output USING btree (substring(locking_bytecode, 0, 26)); +CREATE OR REPLACE FUNCTION search_output (locking_bytecode_hex text[]) + RETURNS SETOF output + LANGUAGE sql STABLE +AS $$ + SELECT * FROM output + -- use output_search_index (first 25 bytes), then compare the full bytecode + WHERE substring(locking_bytecode from 0 for 26) = ANY (ARRAY( + SELECT substring(decode(hex, 'hex') from 1 for 25) FROM unnest(locking_bytecode_hex) AS hex)) + AND locking_bytecode = ANY (ARRAY( + SELECT decode(hex, 'hex') FROM unnest(locking_bytecode_hex) AS hex)) + ORDER BY locking_bytecode ASC +$$; +COMMENT ON FUNCTION search_output (text[]) IS 'Return a list of outputs whose locking bytecode exactly matches any of the provided locking bytecode hex values (of any length, e.g. P2PKH, P2SH20 and P2SH32).'; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 41a794e4..5bcb3e5f 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -82,6 +82,7 @@ const dbUpMigrationPaths = [ ), migration('default/1778437612917_fix_zero_length_pushdata_patterns/up.sql'), migration('default/1778438318512_fix_redeem_bytecode_parser/up.sql'), + migration('default/1791100000000_fix_search_output_exact_matching/up.sql'), ]; const chaingraphInternalApiPort = '3201'; From f728f79c8e1d248dded85b4bd98f1a8efd169069 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 4 Oct 2026 14:18:30 +0100 Subject: [PATCH 42/63] test(sql): search_output_prefix must treat prefix bytes literally search_output_prefix matches with LIKE, so bytes 0x5c (escape), 0x25 (%) and 0x5f (_) in the decoded prefix act as pattern syntax: a prefix containing 0x5c returns nothing and 0x25/0x5f over-match. Prefixes longer than 25 bytes also never match. Add a failing e2e test. Refs #79 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/e2e/e2e.spec.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 5bcb3e5f..675375a0 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -3227,3 +3227,23 @@ test.serial( t.deepEqual(await search([]), []); } ); + +/* cspell: disable */ +test.serial( + '[e2e] [sql] search_output_prefix: treats every byte literally and accepts prefixes longer than 25 bytes', + async (t) => { + const search = async (prefix: string) => + searchFixtureMatches('search_output_prefix($1::text)', prefix); + t.deepEqual(await search(searchFixtureOutputs.backslash), ['backslash']); + t.deepEqual(await search('76a9144444444444444444445c'), ['backslash']); + t.deepEqual(await search('76a91425'), ['percent']); + t.deepEqual(await search('76a9145f'), ['underscore']); + t.deepEqual(await search(searchFixtureOutputs.p2sh32), ['p2sh32']); + t.deepEqual(await search(searchFixtureOutputs.p2pkh), [ + 'p2pkh', + 'p2pkhPrefixed', + ]); + t.deepEqual(await search('aa20'), ['p2sh32', 'p2sh32SamePrefix']); + } +); +/* cspell: enable */ From 23e9788042ed6e5a843dafb0e27ad06455d25ee9 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 4 Oct 2026 14:18:31 +0100 Subject: [PATCH 43/63] fix(sql): match search_output_prefix bytes literally and beyond 25 bytes Replace LIKE with a byte range on the 25-byte expression indexed by output_search_index plus an exact prefix recheck, so bytes 0x5c, 0x25 and 0x5f are matched literally and prefixes longer than 25 bytes work. The index is still used. Fixes #79 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../down.sql | 12 ++++++++++ .../up.sql | 22 +++++++++++++++++++ src/e2e/e2e.spec.ts | 3 +++ 3 files changed, 37 insertions(+) create mode 100644 images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql diff --git a/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/down.sql b/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/down.sql new file mode 100644 index 00000000..0cb663be --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/down.sql @@ -0,0 +1,12 @@ +-- Restore the original search_output_prefix definition from 1616195337538_init. +CREATE OR REPLACE FUNCTION search_output_prefix (locking_bytecode_prefix_hex text) + RETURNS SETOF output + LANGUAGE sql STABLE +AS $$ + SELECT * FROM output + -- use output_locking_bytecode_prefix_index + WHERE substring(locking_bytecode from 0 for 26) + LIKE decode(locking_bytecode_prefix_hex, 'hex') || '%' + ORDER BY locking_bytecode ASC +$$; +COMMENT ON FUNCTION search_output_prefix (text) IS 'Return a list of outputs in which the first 25 bytes of the locking bytecode match the provided prefix hex.'; diff --git a/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql b/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql new file mode 100644 index 00000000..143de68d --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql @@ -0,0 +1,22 @@ +-- search_output_prefix previously matched with LIKE, so bytes 0x5c (escape), +-- 0x25 (%) and 0x5f (_) in the decoded prefix were treated as pattern syntax, +-- and prefixes longer than 25 bytes never matched. Select a byte range on the +-- 25-byte expression indexed by output_search_index, then recheck the exact +-- prefix against the full locking bytecode. +-- Note: this method expects an index: CREATE INDEX output_search_index ON output USING btree (substring(locking_bytecode, 0, 26)); +CREATE OR REPLACE FUNCTION search_output_prefix (locking_bytecode_prefix_hex text) + RETURNS SETOF output + LANGUAGE sql STABLE +AS $$ + SELECT * FROM output + -- use output_search_index: every 25-byte value starting with the prefix lies + -- between the prefix and the prefix padded with 0xff bytes to 25 bytes + WHERE substring(locking_bytecode from 0 for 26) + BETWEEN substring(decode(locking_bytecode_prefix_hex, 'hex') from 1 for 25) + AND substring(decode(locking_bytecode_prefix_hex, 'hex') from 1 for 25) + || decode(repeat('ff', 25 - least(length(decode(locking_bytecode_prefix_hex, 'hex')), 25)), 'hex') + AND substring(locking_bytecode from 1 for length(decode(locking_bytecode_prefix_hex, 'hex'))) + = decode(locking_bytecode_prefix_hex, 'hex') + ORDER BY locking_bytecode ASC +$$; +COMMENT ON FUNCTION search_output_prefix (text) IS 'Return a list of outputs whose locking bytecode begins with the provided prefix hex (of any length; every byte is matched literally).'; diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 675375a0..9b801dfe 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -83,6 +83,9 @@ const dbUpMigrationPaths = [ migration('default/1778437612917_fix_zero_length_pushdata_patterns/up.sql'), migration('default/1778438318512_fix_redeem_bytecode_parser/up.sql'), migration('default/1791100000000_fix_search_output_exact_matching/up.sql'), + migration( + 'default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql' + ), ]; const chaingraphInternalApiPort = '3201'; From 72e813fb15138fc66fd038d7bc5a7ba0ab2cec94 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Thu, 10 Sep 2026 20:55:36 +0100 Subject: [PATCH 44/63] fix(db): bound mempool cleanup conflict lookups to current outpoints --- src/db.ts | 45 ++++-- src/e2e/mempool-cleanup.spec.ts | 243 ++++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 15 deletions(-) create mode 100644 src/e2e/mempool-cleanup.spec.ts diff --git a/src/db.ts b/src/db.ts index c5b28be1..ba7f9cab 100644 --- a/src/db.ts +++ b/src/db.ts @@ -311,6 +311,11 @@ SELECT encode(transaction.hash, 'hex') AS "hash", * replaced by accepted blocks for the same node. This repairs historical rows * missed when block inclusions are added after the node_block trigger has * already fired. + * + * The correlated input lookups use OFFSET 0 as a planning barrier. Without it, + * Postgres can flatten the joins and scan the entire historical input table to + * resolve conflicts for a comparatively small current mempool. Keep each lookup + * constrained to a mempool transaction and then one of its outpoints. */ export const archiveMempoolTransactionsAcceptedByBlocks = async (): Promise< ArchivedMempoolTransaction[] @@ -337,22 +342,32 @@ WITH directly_accepted AS ( replaced_by_accepted AS ( SELECT node_transaction.node_internal_id, node_transaction.transaction_internal_id, - MIN(node_block.accepted_at) AS replaced_at + replacement.replaced_at FROM node_transaction - JOIN input mempool_input - ON mempool_input.transaction_internal_id = node_transaction.transaction_internal_id - JOIN input accepted_input - ON accepted_input.outpoint_transaction_hash = mempool_input.outpoint_transaction_hash - AND accepted_input.outpoint_index = mempool_input.outpoint_index - AND accepted_input.transaction_internal_id != node_transaction.transaction_internal_id - JOIN block_transaction - ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id - JOIN node_block - ON node_block.node_internal_id = node_transaction.node_internal_id - AND node_block.block_internal_id = block_transaction.block_internal_id - WHERE mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea - GROUP BY node_transaction.node_internal_id, - node_transaction.transaction_internal_id + CROSS JOIN LATERAL ( + SELECT MIN(node_block.accepted_at) AS replaced_at + FROM LATERAL ( + SELECT outpoint_transaction_hash, outpoint_index + FROM input + WHERE transaction_internal_id = node_transaction.transaction_internal_id + AND outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea + OFFSET 0 + ) mempool_input + CROSS JOIN LATERAL ( + SELECT transaction_internal_id + FROM input + WHERE outpoint_transaction_hash = mempool_input.outpoint_transaction_hash + AND outpoint_index = mempool_input.outpoint_index + AND transaction_internal_id != node_transaction.transaction_internal_id + OFFSET 0 + ) accepted_input + JOIN block_transaction + ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id + JOIN node_block + ON node_block.node_internal_id = node_transaction.node_internal_id + AND node_block.block_internal_id = block_transaction.block_internal_id + HAVING COUNT(*) > 0 + ) replacement ), archive_candidates AS ( SELECT node_internal_id, transaction_internal_id, replaced_at diff --git a/src/e2e/mempool-cleanup.spec.ts b/src/e2e/mempool-cleanup.spec.ts new file mode 100644 index 00000000..c2b31504 --- /dev/null +++ b/src/e2e/mempool-cleanup.spec.ts @@ -0,0 +1,243 @@ +import { readFileSync } from 'fs'; + +import type { TestFn } from 'ava'; +import baseTest from 'ava'; + +import type * as database from '../db.js'; + +// cspell:words lpad +const test = baseTest as TestFn<{ db: typeof database }>; +const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; +const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; + +/* + * A single connection keeps every operation inside the same temporary schema + * and rollback transaction. No persistent tables are created or modified. + */ +test.before(async (t) => { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = `postgres://chaingraph:very_insecure_postgres_password@${host}:${port}/postgres`; + process.env.CHAINGRAPH_POSTGRES_MAX_CONNECTIONS = '1'; + t.context.db = await import('../db.js'); +}); + +test.beforeEach(async (t) => { + await t.context.db.pool.query(/* sql */ ` +BEGIN; +SET LOCAL search_path = pg_temp; +SET LOCAL statement_timeout = '5s'; +CREATE TEMP TABLE node (internal_id smallint PRIMARY KEY, name text); +CREATE TEMP TABLE transaction (internal_id bigint PRIMARY KEY, hash bytea UNIQUE); +CREATE TEMP TABLE output (transaction_hash bytea, output_index integer, + PRIMARY KEY (transaction_hash, output_index)); +CREATE TEMP TABLE input (transaction_internal_id bigint, input_index integer, + outpoint_transaction_hash bytea, outpoint_index integer, + PRIMARY KEY (transaction_internal_id, input_index)); +CREATE INDEX input_outpoint ON input (outpoint_transaction_hash, outpoint_index); +CREATE TEMP TABLE node_transaction (node_internal_id smallint, + transaction_internal_id bigint, validated_at timestamp DEFAULT '2026-01-01', + PRIMARY KEY (node_internal_id, transaction_internal_id)); +CREATE TEMP TABLE node_transaction_history (node_internal_id smallint, + transaction_internal_id bigint, validated_at timestamp, replaced_at timestamp, + PRIMARY KEY (node_internal_id, transaction_internal_id, validated_at)); +CREATE TEMP TABLE block_transaction (block_internal_id bigint, + transaction_internal_id bigint, + PRIMARY KEY (transaction_internal_id, block_internal_id)); +CREATE TEMP TABLE node_block (node_internal_id smallint, + block_internal_id bigint, accepted_at timestamp NOT NULL, + PRIMARY KEY (node_internal_id, block_internal_id)); +INSERT INTO node VALUES (1, 'alpha'), (2, 'beta'); +INSERT INTO transaction + SELECT id, decode(repeat(lpad(to_hex(id), 2, '0'), 32), 'hex') + FROM generate_series(1, 20) id; +INSERT INTO output SELECT hash, 0 FROM transaction; +`); + const migration = readFileSync( + new URL( + '../../images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql', + import.meta.url + ), + 'utf8' + ); + const cascadeFunction = migration + .split('CREATE TRIGGER')[0]! + .replace( + 'FUNCTION trigger_node_transaction_history_insert()', + 'FUNCTION pg_temp.trigger_node_transaction_history_insert()' + ); + await t.context.db.pool.query(`${cascadeFunction} +CREATE TRIGGER cascade_history AFTER INSERT ON node_transaction_history + REFERENCING NEW TABLE AS new_table FOR EACH STATEMENT + EXECUTE FUNCTION pg_temp.trigger_node_transaction_history_insert(); +`); +}); + +test.afterEach.always(async (t) => { + await t.context.db.pool.query('ROLLBACK;'); +}); + +test.after.always(async (t) => { + await t.context.db.pool.end(); +}); + +const repeatedHashBytes = 32; +const hash = (byte: string) => byte.repeat(repeatedHashBytes); + +const membershipQuery = /* sql */ ` +SELECT node_internal_id AS node, transaction_internal_id::integer AS tx + FROM node_transaction ORDER BY node, tx; +`; +const historyQuery = /* sql */ ` +SELECT node_internal_id AS node, transaction_internal_id::integer AS tx, + replaced_at::text AS replaced + FROM node_transaction_history ORDER BY node, tx; +`; + +test.serial( + '[e2e] cleanup confirms a parent without invalidating descendants or another node', + async (t) => { + const { pool, archiveMempoolTransactionsAcceptedByBlocks: archive } = + t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + SELECT node, tx FROM generate_series(1, 2) node, generate_series(1, 3) tx; +-- Duplicate accepted inclusions must still produce only one history row. +INSERT INTO block_transaction VALUES (1, 1), (2, 1); +INSERT INTO node_block VALUES (1, 1, '2026-01-02'), (1, 2, '2026-01-03'); +`); + t.deepEqual(await archive(), [ + { hash: hash('01'), nodeName: 'alpha', replacedAt: null }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 1, tx: 2 }, + { node: 1, tx: 3 }, + { node: 2, tx: 1 }, + { node: 2, tx: 2 }, + { node: 2, tx: 3 }, + ]); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: null, tx: 1 }, + ]); + t.deepEqual(await archive(), []); + } +); + +test.serial( + '[e2e] cleanup invalidates conflicts and descendants only for the accepting node', + async (t) => { + const { pool, archiveMempoolTransactionsAcceptedByBlocks: archive } = + t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 1, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + SELECT node, tx FROM generate_series(1, 2) node, generate_series(1, 3) tx; +INSERT INTO block_transaction VALUES (1, 4), (2, 4); +INSERT INTO node_block VALUES (1, 1, '2026-01-05'), (1, 2, '2026-01-02'); +`); + t.deepEqual(await archive(), [ + { + hash: hash('01'), + nodeName: 'alpha', + replacedAt: new Date('2026-01-02T00:00:00.000Z'), + }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 2, tx: 1 }, + { node: 2, tx: 2 }, + { node: 2, tx: 3 }, + ]); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: '2026-01-02 00:00:00', tx: 1 }, + { node: 1, replaced: '2026-01-02 00:00:00', tx: 2 }, + { node: 1, replaced: '2026-01-02 00:00:00', tx: 3 }, + ]); + t.deepEqual(await archive(), []); + } +); + +test.serial( + '[e2e] cleanup gives confirmation precedence in mixed confirmation and invalidation batches', + async (t) => { + const { pool, archiveMempoolTransactionsAcceptedByBlocks: archive } = + t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 1, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 8, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 6, 0, hash, 0 FROM transaction WHERE internal_id = 5 + UNION ALL SELECT 7, 0, hash, 0 FROM transaction WHERE internal_id = 6; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 1), (1, 2), (1, 5), (1, 6), (1, 7); +-- Deliberately inconsistent historical acceptance tests NULL precedence. +INSERT INTO block_transaction VALUES (1, 1), (2, 4), (3, 8); +INSERT INTO node_block VALUES + (1, 1, '2026-01-02'), (1, 2, '2026-01-03'), (1, 3, '2026-01-04'); +`); + t.deepEqual(await archive(), [ + { hash: hash('01'), nodeName: 'alpha', replacedAt: null }, + { + hash: hash('05'), + nodeName: 'alpha', + replacedAt: new Date('2026-01-04T00:00:00.000Z'), + }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, [{ node: 1, tx: 2 }]); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: null, tx: 1 }, + { node: 1, replaced: '2026-01-04 00:00:00', tx: 5 }, + { node: 1, replaced: '2026-01-04 00:00:00', tx: 6 }, + { node: 1, replaced: '2026-01-04 00:00:00', tx: 7 }, + ]); + } +); + +test.serial( + '[e2e] cleanup ignores other-node confirmations, self, coinbase and unconfirmed conflicts', + async (t) => { + const { pool, archiveMempoolTransactionsAcceptedByBlocks: archive } = + t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 1, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 2, 0, decode(repeat('00', 32), 'hex'), 0 + UNION ALL SELECT 3, 0, decode(repeat('00', 32), 'hex'), 0 + UNION ALL SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 6, 0, hash, 0 FROM transaction WHERE internal_id = 12 + UNION ALL SELECT 7, 0, hash, 0 FROM transaction WHERE internal_id = 12 + UNION ALL SELECT 8, 0, hash, 1 FROM transaction WHERE internal_id = 13 + UNION ALL SELECT 9, 0, hash, 0 FROM transaction WHERE internal_id = 13; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 1), (1, 2), (1, 4), (1, 6), (1, 8); +INSERT INTO block_transaction VALUES (1, 1), (2, 3), (3, 5), (4, 9); +INSERT INTO node_block VALUES + (2, 1, '2026-01-02'), (1, 2, '2026-01-02'), + (2, 3, '2026-01-02'), (1, 4, '2026-01-02'); +`); + t.deepEqual(await archive(), []); + t.deepEqual((await pool.query(historyQuery)).rows, []); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 1, tx: 1 }, + { node: 1, tx: 2 }, + { node: 1, tx: 4 }, + { node: 1, tx: 6 }, + { node: 1, tx: 8 }, + ]); + } +); + +test.serial('[e2e] cleanup accepts an empty mempool', async (t) => { + t.deepEqual( + await t.context.db.archiveMempoolTransactionsAcceptedByBlocks(), + [] + ); +}); From 704aa3be285a23faa563d16fd928cdeb6bb09006 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Tue, 6 Oct 2026 23:20:32 +0100 Subject: [PATCH 45/63] test(sql): assert search_output uses output_search_index With sequential scans disabled, EXPLAIN both search_output and search_output_prefix against a copy of the managed output_search_index (substring(locking_bytecode, 0, 26)) and require the plans to use it, so a mismatch between the function expression and the index expression shows up as a Seq Scan. Adapted from the dropped 5cff1be without the 1-based rewrite or array-era indexes. Refs #78 Co-Authored-By: Claude Fable 5.1 --- src/e2e/e2e.spec.ts | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 9b801dfe..1d68404f 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -27,6 +27,7 @@ import { execa } from 'execa'; import got from 'got'; import pg from 'pg'; +import { indexDefinitions } from '../components/db-utils.js'; import type { ChaingraphTransaction } from '../types/chaingraph.js'; import { chaingraphE2eLogPath, logger } from './e2e.spec.logging.helper.js'; @@ -3250,3 +3251,61 @@ test.serial( } ); /* cspell: enable */ + +test.serial( + '[e2e] [sql] search functions use the 25-byte locking bytecode prefix index', + async (t) => { + const prefixIndexes = Object.entries(indexDefinitions).filter( + ([, definition]) => definition.includes('locking_bytecode') + ); + t.deepEqual( + prefixIndexes.map(([name]) => name), + ['output_search_index'] + ); + prefixIndexes.forEach(([name, definition]) => { + t.true(definition.includes('substring(locking_bytecode, 0, 26)'), name); + }); + await client.query('BEGIN;'); + const explainPlans = async () => { + await prefixIndexes.reduce>( + async (chain, [name, definition]) => + chain.then(async () => + client.query( + definition.replace( + `CREATE INDEX ${name}`, + `CREATE INDEX test_${name}` + ) + ) + ), + Promise.resolve(undefined) + ); + // cspell: disable-next-line + await client.query('SET LOCAL enable_seqscan = off;'); + const plan = async (query: string) => + ( + await client.query<{ [column: string]: string }>( + `EXPLAIN (COSTS OFF) ${query}` + ) + ).rows + .map((row) => Object.values(row).join('')) + .join('\n'); + return [ + // cspell: disable-next-line + await plan(`SELECT * FROM search_output(ARRAY['76a91411'])`), + // cspell: disable-next-line + await plan(`SELECT * FROM search_output_prefix('76a914')`), + ]; + }; + const rollback = async () => client.query('ROLLBACK;'); + const plans = await explainPlans().then( + async (result) => rollback().then(() => result), + async (error: unknown) => + // eslint-disable-next-line functional/no-promise-reject -- roll back, then propagate the original failure + rollback().then(async () => Promise.reject(error)) + ); + plans.forEach((plan) => { + t.true(plan.includes('test_output_search_index'), plan); + t.false(plan.includes('Seq Scan'), plan); + }); + } +); From 80ed197898bd6a8a59f88296409deb9edd83b305 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 12 Oct 2026 13:51:54 +0100 Subject: [PATCH 46/63] fix(db): never expire a mempool transaction already accepted by a block The agent's mempool scan archives node_transaction rows accepted (or replaced) by blocks, then reads rows expiring soon and arms an exact expiry timer for each. A row inserted between those two statements with an old validated_at (e.g. a confirmed transaction re-announced or re-validated) got an immediate timer and was archived to node_transaction_history with replaced_at = expiry, recording a confirmed transaction as dropped. The e2e test "archives stale mempool transactions already accepted by blocks" hit this intermittently. archiveMempoolTransaction now guards its DELETE with NOT EXISTS checks for an inclusion of the transaction, or of a conflicting spend of one of its outpoints, in a block accepted by the same node. The guard is evaluated in the archiving statement itself, so no earlier read can go stale. Such rows are left for archiveMempoolTransactionsAcceptedByBlocks, which records replaced_at NULL (confirmed) or the block's accepted_at (replaced) on the next scan. Adds a deterministic regression test that inserts rows between the two scan steps and fires the expiry archive directly. Co-Authored-By: Claude Opus 5.5 --- src/agent.ts | 2 +- src/db.ts | 36 ++++++++++++- src/e2e/mempool-cleanup.spec.ts | 90 +++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index be2b08bd..779280d4 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -1111,7 +1111,7 @@ export class Agent { }); if (archivedCount === 0) { this.logger.debug( - `Agent: skipped expiry of mempool transaction ${transaction.hash} for node ${transaction.nodeName}; it has already left node_transaction.` + `Agent: skipped expiry of mempool transaction ${transaction.hash} for node ${transaction.nodeName}; it has already left node_transaction, or it (or a conflicting spend) is in a block accepted by the node and is left for the accepted-block archive.` ); return; } diff --git a/src/db.ts b/src/db.ts index ba7f9cab..3b1525bf 100644 --- a/src/db.ts +++ b/src/db.ts @@ -426,8 +426,17 @@ SELECT encode(transaction.hash, 'hex') AS "hash", }; /** - * Archive a single node_transaction row. Existing history triggers handle any - * same-node descendants with the same replaced_at timestamp. + * Archive a single expired node_transaction row. Existing history triggers + * handle any same-node descendants with the same replaced_at timestamp. + * + * The row is left in place (archivedCount 0) if the transaction, or a + * conflicting spend of one of its outpoints, is already in a block accepted by + * the same node: {@link archiveMempoolTransactionsAcceptedByBlocks} archives + * those with the correct replaced_at (NULL or the block's accepted_at). The + * guard is part of the DELETE so it is evaluated against the row being + * archived, not a read taken earlier: a row inserted after the agent's + * accepted-block archive but before its expiry scan would otherwise be + * archived as expired/replaced even though it is confirmed. */ export const archiveMempoolTransaction = async ({ nodeInternalId, @@ -449,6 +458,29 @@ WITH deleted_row AS ( DELETE FROM node_transaction WHERE node_internal_id = $1 AND transaction_internal_id = $2 + AND NOT EXISTS ( + SELECT 1 + FROM block_transaction + JOIN node_block + ON node_block.node_internal_id = $1 + AND node_block.block_internal_id = block_transaction.block_internal_id + WHERE block_transaction.transaction_internal_id = $2 + ) + AND NOT EXISTS ( + SELECT 1 + FROM input mempool_input + JOIN input accepted_input + ON accepted_input.outpoint_transaction_hash = mempool_input.outpoint_transaction_hash + AND accepted_input.outpoint_index = mempool_input.outpoint_index + AND accepted_input.transaction_internal_id != $2 + JOIN block_transaction + ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id + JOIN node_block + ON node_block.node_internal_id = $1 + AND node_block.block_internal_id = block_transaction.block_internal_id + WHERE mempool_input.transaction_internal_id = $2 + AND mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea + ) RETURNING node_internal_id, transaction_internal_id, validated_at, diff --git a/src/e2e/mempool-cleanup.spec.ts b/src/e2e/mempool-cleanup.spec.ts index c2b31504..b26dd39d 100644 --- a/src/e2e/mempool-cleanup.spec.ts +++ b/src/e2e/mempool-cleanup.spec.ts @@ -241,3 +241,93 @@ test.serial('[e2e] cleanup accepts an empty mempool', async (t) => { [] ); }); + +/* + * Regression: the agent's mempool scan runs archiveMempoolTransactionsAcceptedByBlocks + * and then getMempoolTransactionsExpiringBefore as two separate statements, and + * expires the rows returned by the second via archiveMempoolTransaction. A row + * inserted between the two (e.g. an announcement of an already-confirmed + * transaction with an old validated_at) used to be archived as expired, with + * replaced_at = expiry, although the node had accepted it in a block. Expiry + * must leave such rows for the accepted-block archive. + */ +test.serial( + '[e2e] expiry never archives a transaction accepted or replaced by a block for that node', + async (t) => { + const { + pool, + archiveMempoolTransaction, + archiveMempoolTransactionsAcceptedByBlocks: archive, + getMempoolTransactionsExpiringBefore, + } = t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 10; +-- tx 1 is confirmed and tx 5 (conflicting with tx 4) is mined, for alpha only. +INSERT INTO block_transaction VALUES (1, 1), (2, 5); +INSERT INTO node_block VALUES (1, 1, '2026-01-02'), (1, 2, '2026-01-03'); +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 3); +`); + // Scan step 1: the accepted-block archive finds nothing to do yet. + t.deepEqual(await archive(), []); + // A concurrent insert lands between the two scan steps. + await pool.query(/* sql */ ` +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 1), (1, 4), (2, 1), (2, 4); +`); + // Scan step 2: every row is already past expiry, so all get timers. + const expiring = await getMempoolTransactionsExpiringBefore({ + expirationMs: 3_600_000, + expiresBefore: new Date('2026-01-10T00:00:00.000Z'), + }); + t.deepEqual( + expiring.map((row) => ({ + node: row.nodeName, + tx: row.transactionInternalId, + })), + [ + { node: 'alpha', tx: 1 }, + { node: 'alpha', tx: 3 }, + { node: 'alpha', tx: 4 }, + { node: 'beta', tx: 1 }, + { node: 'beta', tx: 4 }, + ] + ); + // Timers fire: only rows not resolved by a block accepted by that node expire. + const archivedCounts = await expiring.reduce>( + async (previous, row) => [ + ...(await previous), + await archiveMempoolTransaction({ + nodeInternalId: row.nodeInternalId, + replacedAt: row.expiresAt, + transactionInternalId: row.transactionInternalId, + }), + ], + Promise.resolve([]) + ); + t.deepEqual(archivedCounts, [0, 1, 0, 1, 1]); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 1, tx: 1 }, + { node: 1, tx: 4 }, + ]); + // The next scan archives them with the block-derived replaced_at. + t.deepEqual(await archive(), [ + { hash: hash('01'), nodeName: 'alpha', replacedAt: null }, + { + hash: hash('04'), + nodeName: 'alpha', + replacedAt: new Date('2026-01-03T00:00:00.000Z'), + }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, []); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: null, tx: 1 }, + { node: 1, replaced: '2026-01-01 01:00:00', tx: 3 }, + { node: 1, replaced: '2026-01-03 00:00:00', tx: 4 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 4 }, + ]); + } +); From 6749fefc7db98d707f8c5461de3a1eed50d6f25f Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Sun, 11 Oct 2026 21:01:15 +0100 Subject: [PATCH 47/63] test(sql): accept either copy of output_search_index in the search plan test The test creates test_output_search_index next to the managed output_search_index. The two are identical, so the planner's choice is a tie that flips once Postgres has auto-analyzed output (the managed index then has expression statistics, the fresh copy none): reproducible on this branch's e2e runs, passing on paryon only by timing. Either index satisfies what the test checks (the 25-byte prefix index, no sequential scan). Co-Authored-By: Claude Opus 5.5 --- src/e2e/e2e.spec.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 1d68404f..c7ea108f 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -3303,8 +3303,13 @@ test.serial( // eslint-disable-next-line functional/no-promise-reject -- roll back, then propagate the original failure rollback().then(async () => Promise.reject(error)) ); + /* + * Either copy: the two are identical, and once Postgres has auto-analyzed + * `output` the managed index has expression statistics the fresh copy + * lacks, which tips the planner's tie towards it. + */ plans.forEach((plan) => { - t.true(plan.includes('test_output_search_index'), plan); + t.true(/\b(?:test_)?output_search_index\b/u.test(plan), plan); t.false(plan.includes('Seq Scan'), plan); }); } From 740672ba3244669f8e3259186988ee42d4833468 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Tue, 13 Oct 2026 07:25:46 +0100 Subject: [PATCH 48/63] fix(db): serialize mempool expiry so a stale mempool cannot deadlock Archiving an expired mempool transaction also archives its same-node descendants (the node_transaction_history trigger's recursive cascade). The agent fired one archive statement per expiry timer, racing on every pool connection: a parent's expiry locked its own row and then waited in the cascade for a descendant's row, while the descendant's own expiry held that row and waited (in its cascade) for a row the first had locked. A restarted agent whose 6-day-stale mempool expired all at once hit >1,300 deadlocks (40P01) and made no net progress (2026-10-13 backfill lab). - archiveMempoolTransactions: one set-based statement per batch; takes a transaction-level advisory lock before any row lock (serializes expiry across agents sharing the database), locks rows in (node, transaction) order, keeps the accepted-by-block guards, and retries a deadlock or serialization failure against other writers with exponential backoff. - expireMempoolTransaction: in-process queue; timers firing together are archived in batches of up to 500, one batch at a time on one connection. - archiveMempoolTransaction keeps its signature (a one-row batch). - New spec reproduces the lab: 60 x 40 chained/fanned-out mempool transactions for two nodes, all expiring at once on a 16-connection pool (failed with dozens of 40P01 before; passes 5/5 now). Co-Authored-By: Claude Opus 5.5 --- src/agent.ts | 7 +- src/db.ts | 273 +++++++++++++++++---- src/e2e/mempool-expiry-concurrency.spec.ts | 240 ++++++++++++++++++ 3 files changed, 470 insertions(+), 50 deletions(-) create mode 100644 src/e2e/mempool-expiry-concurrency.spec.ts diff --git a/src/agent.ts b/src/agent.ts index 779280d4..ddd25dee 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -39,9 +39,9 @@ import { } from './config.js'; import { acceptBlocksViaHeaders, - archiveMempoolTransaction, archiveMempoolTransactionsAcceptedByBlocks, createIndexes, + expireMempoolTransaction, getAllKnownBlockHashes, getIncompleteBlocks, getIndexCreationProgress, @@ -1104,12 +1104,13 @@ export class Agent { } async expireMempoolTransaction(transaction: ExpiringMempoolTransaction) { - const archivedCount = await archiveMempoolTransaction({ + // queued: expiries run one ordered batch at a time (see expireMempoolTransaction in db.ts) + const archived = await expireMempoolTransaction({ nodeInternalId: transaction.nodeInternalId, replacedAt: transaction.expiresAt, transactionInternalId: transaction.transactionInternalId, }); - if (archivedCount === 0) { + if (!archived) { this.logger.debug( `Agent: skipped expiry of mempool transaction ${transaction.hash} for node ${transaction.nodeName}; it has already left node_transaction, or it (or a conflicting spend) is in a block accepted by the node and is left for the accepted-block archive.` ); diff --git a/src/db.ts b/src/db.ts index 3b1525bf..82f9b8e6 100644 --- a/src/db.ts +++ b/src/db.ts @@ -425,46 +425,99 @@ SELECT encode(transaction.hash, 'hex') AS "hash", } }; +export interface MempoolTransactionExpiry { + nodeInternalId: number; + replacedAt: Date; + transactionInternalId: number; +} + +// cspell:words xact unnest +const deadlockDetectedCode = '40P01'; +const serializationFailureCode = '40001'; +const isRetriedConflict = (err: unknown) => + [deadlockDetectedCode, serializationFailureCode].includes( + (err as { code?: string } | undefined)?.code ?? '' + ); +const expiryRetryLimit = 6; +const expiryRetryBaseMs = 50; +const expiryBatchSize = 500; +/** + * Advisory lock key serializing mempool expiry across every agent sharing the + * database (the agent also runs expiry batches one at a time). + */ +const expiryAdvisoryLockKey = 1970172700; + /** - * Archive a single expired node_transaction row. Existing history triggers - * handle any same-node descendants with the same replaced_at timestamp. + * Archive expired node_transaction rows in one statement. Existing history + * triggers handle any same-node descendants: those not in the batch get the + * replaced_at of their archived ancestor. * - * The row is left in place (archivedCount 0) if the transaction, or a - * conflicting spend of one of its outpoints, is already in a block accepted by - * the same node: {@link archiveMempoolTransactionsAcceptedByBlocks} archives - * those with the correct replaced_at (NULL or the block's accepted_at). The - * guard is part of the DELETE so it is evaluated against the row being - * archived, not a read taken earlier: a row inserted after the agent's - * accepted-block archive but before its expiry scan would otherwise be + * A row is left in place (not returned) if the transaction, or a conflicting + * spend of one of its outpoints, is already in a block accepted by the same + * node: {@link archiveMempoolTransactionsAcceptedByBlocks} archives those with + * the correct replaced_at (NULL or the block's accepted_at). The guard is part + * of the statement that locks and deletes the row, so it is evaluated against + * the row being archived, not a read taken earlier: a row inserted after the + * agent's accepted-block archive but before its expiry scan would otherwise be * archived as expired/replaced even though it is confirmed. + * + * Deadlocks: archiving a mempool transaction also archives its descendants + * (the history trigger's recursive cascade), so concurrent single-row expiries + * of a parent and of its descendants lock the same rows in opposite orders + * (each locks its own row, then the cascade waits for the other's) and + * deadlock; a restarted agent's stale mempool, expiring all at once on every + * pool connection, live-locked this way. Expiry therefore runs one batch at a + * time (see {@link expireMempoolTransaction}), under a transaction-level + * advisory lock (also across agents sharing the database), locks its rows in (node, transaction) order before deleting + * them, and retries a deadlock or serialization failure against other writers + * (block acceptance, double-spend replacement) with backoff. */ -export const archiveMempoolTransaction = async ({ - nodeInternalId, - replacedAt, - transactionInternalId, -}: { - nodeInternalId: number; - replacedAt: Date; - transactionInternalId: number; -}) => { - const client = await pool.connect(); - // eslint-disable-next-line functional/no-try-statement - try { - const result = await client.query<{ - archivedCount: number; - }>( - /* sql */ ` -WITH deleted_row AS ( - DELETE FROM node_transaction - WHERE node_internal_id = $1 - AND transaction_internal_id = $2 - AND NOT EXISTS ( +export const archiveMempoolTransactions = async ( + expiries: MempoolTransactionExpiry[] +): Promise => { + if (expiries.length === 0) { + return []; + } + const sorted = [...expiries].sort( + (a, b) => + a.nodeInternalId - b.nodeInternalId || + a.transactionInternalId - b.transactionInternalId + ); + const archiveOnce = async () => { + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const result = await client.query<{ + nodeInternalId: string; + transactionInternalId: string; + }>( + /* sql */ ` +WITH serialized AS ( + -- volatile, so materialized: scanned (and the lock taken) before any + -- joined row can be produced, i.e. before any row lock below + SELECT pg_advisory_xact_lock($4::bigint) +), +expiry AS ( + SELECT node_internal_id, transaction_internal_id, replaced_at + FROM serialized + CROSS JOIN unnest($1::smallint[], $2::bigint[], $3::timestamp[]) + AS expiry (node_internal_id, transaction_internal_id, replaced_at) +), +locked AS ( + SELECT node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + expiry.replaced_at + FROM expiry + JOIN node_transaction + ON node_transaction.node_internal_id = expiry.node_internal_id + AND node_transaction.transaction_internal_id = expiry.transaction_internal_id + WHERE NOT EXISTS ( SELECT 1 FROM block_transaction JOIN node_block - ON node_block.node_internal_id = $1 + ON node_block.node_internal_id = expiry.node_internal_id AND node_block.block_internal_id = block_transaction.block_internal_id - WHERE block_transaction.transaction_internal_id = $2 + WHERE block_transaction.transaction_internal_id = expiry.transaction_internal_id ) AND NOT EXISTS ( SELECT 1 @@ -472,36 +525,162 @@ WITH deleted_row AS ( JOIN input accepted_input ON accepted_input.outpoint_transaction_hash = mempool_input.outpoint_transaction_hash AND accepted_input.outpoint_index = mempool_input.outpoint_index - AND accepted_input.transaction_internal_id != $2 + AND accepted_input.transaction_internal_id != expiry.transaction_internal_id JOIN block_transaction ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id JOIN node_block - ON node_block.node_internal_id = $1 + ON node_block.node_internal_id = expiry.node_internal_id AND node_block.block_internal_id = block_transaction.block_internal_id - WHERE mempool_input.transaction_internal_id = $2 + WHERE mempool_input.transaction_internal_id = expiry.transaction_internal_id AND mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea ) - RETURNING node_internal_id, - transaction_internal_id, - validated_at, - ${dateToTimestampWithoutTimezone(replacedAt)} AS replaced_at + ORDER BY node_transaction.node_internal_id, node_transaction.transaction_internal_id + FOR UPDATE OF node_transaction +), +deleted_rows AS ( + DELETE FROM node_transaction + USING locked + WHERE node_transaction.node_internal_id = locked.node_internal_id + AND node_transaction.transaction_internal_id = locked.transaction_internal_id + RETURNING node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + node_transaction.validated_at, + locked.replaced_at ), inserted_history AS ( INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at - FROM deleted_row - RETURNING transaction_internal_id + FROM deleted_rows + RETURNING node_internal_id, transaction_internal_id ) -SELECT COUNT(*)::integer AS "archivedCount" FROM inserted_history; +SELECT node_internal_id::text AS "nodeInternalId", + transaction_internal_id::text AS "transactionInternalId" + FROM inserted_history; `, - [nodeInternalId, transactionInternalId] - ); - return result.rows[0]!.archivedCount; - } finally { - client.release(); + [ + sorted.map((expiry) => expiry.nodeInternalId), + sorted.map((expiry) => expiry.transactionInternalId), + sorted.map((expiry) => expiry.replacedAt.toISOString()), + expiryAdvisoryLockKey, + ] + ); + const archived = new Set( + result.rows.map( + (row) => `${row.nodeInternalId}:${row.transactionInternalId}` + ) + ); + return sorted.filter((expiry) => + archived.has(`${expiry.nodeInternalId}:${expiry.transactionInternalId}`) + ); + } finally { + client.release(); + } + }; + const attempt = async (retry: number): Promise => + archiveOnce().catch(async (err: unknown) => { + if (!isRetriedConflict(err) || retry >= expiryRetryLimit) { + // eslint-disable-next-line functional/no-throw-statement + throw err; + } + // eslint-disable-next-line @typescript-eslint/no-magic-numbers + const backoffMs = expiryRetryBaseMs * 2 ** retry * (1 + Math.random()); + await new Promise((resolve) => { + setTimeout(resolve, backoffMs); + }); + return attempt(retry + 1); + }); + return attempt(0); +}; + +/** + * Archive a single expired node_transaction row (see + * {@link archiveMempoolTransactions}); returns the number archived (0 or 1). + */ +export const archiveMempoolTransaction = async ( + expiry: MempoolTransactionExpiry +) => (await archiveMempoolTransactions([expiry])).length; + +const pendingExpiries = new Map< + string, + { + expiry: MempoolTransactionExpiry; + settle: { + reject: (err: unknown) => void; + resolve: (archived: boolean) => void; + }[]; } +>(); +// eslint-disable-next-line functional/no-let, @typescript-eslint/init-declarations +let expiryDrain: Promise | undefined; + +const drainExpiries = async (): Promise => { + const batch = [...pendingExpiries.entries()].slice(0, expiryBatchSize); + if (batch.length === 0) { + return undefined; + } + batch.forEach(([key]) => pendingExpiries.delete(key)); + await archiveMempoolTransactions(batch.map(([, entry]) => entry.expiry)) + .then((archived) => { + const archivedKeys = new Set( + archived.map( + (expiry) => `${expiry.nodeInternalId}:${expiry.transactionInternalId}` + ) + ); + batch.forEach(([key, entry]) => { + entry.settle.forEach(({ resolve }) => { + resolve(archivedKeys.has(key)); + }); + }); + }) + .catch((err: unknown) => { + batch.forEach(([, entry]) => { + entry.settle.forEach(({ reject }) => { + reject(err); + }); + }); + }); + return drainExpiries(); }; +const startExpiryDrain = () => { + if (expiryDrain !== undefined) { + return; + } + expiryDrain = new Promise((started) => { + // let every timer firing in this tick join the first batch + setImmediate(started); + }) + .then(drainExpiries) + .finally(() => { + expiryDrain = undefined; + if (pendingExpiries.size > 0) { + startExpiryDrain(); + } + }); +}; + +/** + * Queue an expired mempool transaction for archiving; resolves `true` if it was + * archived, `false` if it was left in place (already gone, or accepted by a + * block for the node). Expiries run one batch at a time (up to + * {@link expiryBatchSize} rows per statement): when many fire at once (a + * restarted agent's stale mempool) they are archived in a few ordered + * statements on one connection instead of one racing statement per pool + * connection. + */ +export const expireMempoolTransaction = async ( + expiry: MempoolTransactionExpiry +) => + new Promise((resolve, reject) => { + const key = `${expiry.nodeInternalId}:${expiry.transactionInternalId}`; + const pending = pendingExpiries.get(key); + if (pending === undefined) { + pendingExpiries.set(key, { expiry, settle: [{ reject, resolve }] }); + } else { + pending.settle.push({ reject, resolve }); + } + startExpiryDrain(); + }); /** * Create or update one or more trusted node in the Chaingraph database, * returning it's internal ID. diff --git a/src/e2e/mempool-expiry-concurrency.spec.ts b/src/e2e/mempool-expiry-concurrency.spec.ts new file mode 100644 index 00000000..509a2fd0 --- /dev/null +++ b/src/e2e/mempool-expiry-concurrency.spec.ts @@ -0,0 +1,240 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import { readFileSync } from 'fs'; + +import type { ExecutionContext, TestFn } from 'ava'; +import baseTest from 'ava'; + +import type * as database from '../db.js'; + +// cspell:words lpad +const test = baseTest as TestFn<{ db: typeof database; schema: string }>; +const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; +const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; +const schema = `expiry_concurrency_${process.pid}`; +const poolSize = 16; + +/* + * Real (committed) tables in a scratch schema, so many pool connections see the + * same rows and take real row locks: the stale-mempool expiry of a restarted + * agent fires one timer per expired transaction at once. + */ +test.before(async (t) => { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = `postgres://chaingraph:very_insecure_postgres_password@${host}:${port}/postgres?options=${encodeURIComponent( + `-c search_path=${schema} -c deadlock_timeout=50ms` + )}`; + process.env.CHAINGRAPH_POSTGRES_MAX_CONNECTIONS = `${poolSize}`; + t.context.db = await import('../db.js'); + t.context.schema = schema; + const migration = readFileSync( + new URL( + '../../images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql', + import.meta.url + ), + 'utf8' + ); + const cascadeFunction = migration.split('CREATE TRIGGER')[0]!; + await t.context.db.pool.query(/* sql */ ` +DROP SCHEMA IF EXISTS ${schema} CASCADE; +CREATE SCHEMA ${schema}; +SET search_path = ${schema}; +CREATE TABLE node (internal_id smallint PRIMARY KEY, name text); +CREATE TABLE transaction (internal_id bigint PRIMARY KEY, hash bytea UNIQUE); +CREATE TABLE output (transaction_hash bytea, output_index integer, + PRIMARY KEY (transaction_hash, output_index)); +CREATE TABLE input (transaction_internal_id bigint, input_index integer, + outpoint_transaction_hash bytea, outpoint_index integer, + PRIMARY KEY (transaction_internal_id, input_index)); +CREATE INDEX input_outpoint ON input (outpoint_transaction_hash, outpoint_index); +CREATE TABLE node_transaction (node_internal_id smallint, + transaction_internal_id bigint, validated_at timestamp NOT NULL, + PRIMARY KEY (node_internal_id, transaction_internal_id)); +CREATE TABLE node_transaction_history (node_internal_id smallint, + transaction_internal_id bigint, validated_at timestamp, replaced_at timestamp, + PRIMARY KEY (node_internal_id, transaction_internal_id, validated_at)); +CREATE TABLE block_transaction (block_internal_id bigint, + transaction_internal_id bigint, + PRIMARY KEY (transaction_internal_id, block_internal_id)); +CREATE TABLE node_block (node_internal_id smallint, + block_internal_id bigint, accepted_at timestamp NOT NULL, + PRIMARY KEY (node_internal_id, block_internal_id)); +${cascadeFunction} +CREATE TRIGGER cascade_history AFTER INSERT ON node_transaction_history + REFERENCING NEW TABLE AS new_table FOR EACH STATEMENT + EXECUTE FUNCTION trigger_node_transaction_history_insert(); +`); +}); + +test.after.always(async (t) => { + await t.context.db.pool.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE;`); + await t.context.db.pool.end(); +}); + +const chains = 60; +const depth = 40; + +/** + * `chains` chains of `depth` mempool transactions (each spends output 0 of the + * previous one), every transaction also fanning out to the next chain's + * transaction at the same depth, all validated at the same moment for two + * nodes: a restarted agent's stale mempool, all expiring at once. + */ +const createStaleMempool = async (db: typeof database) => { + await db.pool.query(/* sql */ ` +TRUNCATE node, transaction, output, input, node_transaction, + node_transaction_history, block_transaction, node_block; +INSERT INTO node VALUES (1, 'alpha'), (2, 'beta'); +INSERT INTO transaction + SELECT id, sha256(int8send(id)) FROM generate_series(1, ${chains * depth}) id; +INSERT INTO output SELECT hash, o FROM transaction, generate_series(0, 1) o; +-- tx (c, d) has internal id c * depth + d + 1 +INSERT INTO input + SELECT c * ${depth} + d + 1, 0, parent.hash, 0 + FROM generate_series(0, ${chains - 1}) c CROSS JOIN generate_series(1, ${ + depth - 1 + }) d + JOIN transaction parent ON parent.internal_id = c * ${depth} + d + UNION ALL + SELECT c * ${depth} + d + 1, 1, parent.hash, 1 + FROM generate_series(1, ${chains - 1}) c CROSS JOIN generate_series(0, ${ + depth - 1 + }) d + JOIN transaction parent ON parent.internal_id = (c - 1) * ${depth} + d + 1; +INSERT INTO node_transaction + SELECT node.internal_id, transaction.internal_id, '2026-01-01' + FROM node CROSS JOIN transaction; +`); +}; + +const countRows = async (db: typeof database) => { + const result = await db.pool.query<{ + current: string; + history: string; + }>(/* sql */ ` +SELECT (SELECT count(*) FROM node_transaction)::text AS current, + (SELECT count(*) FROM node_transaction_history)::text AS history;`); + return { + current: Number(result.rows[0]!.current), + history: Number(result.rows[0]!.history), + }; +}; + +const shuffled = (items: T[], seed: number) => { + const copy = [...items]; + // eslint-disable-next-line functional/no-let + let state = seed; + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let i = copy.length - 1; i > 0; i -= 1) { + // eslint-disable-next-line no-bitwise + state = (state * 1103515245 + 12345) & 0x7fffffff; + const j = state % (i + 1); + [copy[i], copy[j]] = [copy[j]!, copy[i]!]; + } + return copy; +}; + +const expectAllArchived = async ( + t: ExecutionContext<{ db: typeof database; schema: string }>, + errors: unknown[] +) => { + t.deepEqual( + errors.map((err) => (err as { code?: string }).code ?? err), + [], + 'no expiry fails (40P01 = deadlock)' + ); + t.deepEqual(await countRows(t.context.db), { + current: 0, + history: chains * depth * 2, + }); + const replacedAt = await t.context.db.pool.query<{ replaced: string }>( + /* sql */ `SELECT DISTINCT replaced_at::text AS replaced FROM node_transaction_history;` + ); + t.deepEqual(replacedAt.rows, [{ replaced: '2026-01-01 01:00:00' }]); +}; + +const staleExpiries = async (db: typeof database) => { + await createStaleMempool(db); + const expiring = await db.getMempoolTransactionsExpiringBefore({ + expirationMs: 3_600_000, + expiresBefore: new Date('2026-01-10T00:00:00.000Z'), + }); + return shuffled(expiring, 7).map((row) => ({ + nodeInternalId: row.nodeInternalId, + replacedAt: row.expiresAt, + transactionInternalId: row.transactionInternalId, + })); +}; + +/* + * Regression (2026-10-13 backfill lab): a restarted agent's 6-day-stale mempool + * expired all at once, one archive statement per timer on every pool + * connection. Archiving a transaction cascades to its descendants (history + * trigger), so a parent's and a descendant's expiry locked the same rows in + * opposite orders: >1,300 deadlocks and no net progress. Before the fix this + * test failed with dozens of 40P01 errors (or timed out). + */ +test.serial( + '[e2e] concurrent single-row expiries of a stale mempool archive every row without deadlocks', + async (t) => { + const { db } = t.context; + const expiries = await staleExpiries(db); + t.is(expiries.length, chains * depth * 2); + const errors: unknown[] = []; + await Promise.all( + expiries.map(async (expiry) => + db.archiveMempoolTransaction(expiry).catch((err: unknown) => { + errors.push(err); + }) + ) + ); + await expectAllArchived(t, errors); + } +); + +test.serial( + '[e2e] concurrent expiry batches serialize and archive every row', + async (t) => { + const { db } = t.context; + const expiries = await staleExpiries(db); + const batchCount = poolSize; + const errors: unknown[] = []; + await Promise.all( + Array.from({ length: batchCount }, async (_, batch) => + db + .archiveMempoolTransactions( + expiries.filter((_expiry, index) => index % batchCount === batch) + ) + .catch((err: unknown) => { + errors.push(err); + }) + ) + ); + await expectAllArchived(t, errors); + } +); + +test.serial( + '[e2e] the expiry queue archives a stale mempool firing at once in a few ordered batches', + async (t) => { + const { db } = t.context; + const expiries = await staleExpiries(db); + const errors: unknown[] = []; + const results = await Promise.all( + expiries.map(async (expiry) => + db.expireMempoolTransaction(expiry).catch((err: unknown) => { + errors.push(err); + return undefined; + }) + ) + ); + await expectAllArchived(t, errors); + const archivedDirectly = results.filter((archived) => archived === true); + t.true(archivedDirectly.length > 0); + t.is( + archivedDirectly.length + + results.filter((archived) => archived === false).length, + expiries.length + ); + // a duplicate request for an already-archived row resolves false + t.false(await db.expireMempoolTransaction(expiries[0]!)); + } +); From 624a2dedc4c07765fedc44db7f0e01629f011baf Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 19 Oct 2026 11:37:36 +0100 Subject: [PATCH 49/63] ci: use current major versions of GitHub actions Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yaml | 12 ++++++------ .github/workflows/image-agent.yaml | 8 ++++---- .github/workflows/image-bchn.yaml | 8 ++++---- .github/workflows/image-hasura.yaml | 8 ++++---- .github/workflows/release-chart.yaml | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index efa97ce1..3601f263 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,22 +20,22 @@ jobs: - 5432:5432 steps: - name: Check out repository code - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: submodules: 'true' - name: Install Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v7 with: node-version: '18' - run: yarn install --immutable --immutable-cache - run: yarn test - run: yarn cov:lcov - name: Upload test coverage - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} - name: Save E2E test logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v7 if: always() with: name: e2e-test-log @@ -45,11 +45,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository code - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: submodules: 'true' - name: Install Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v7 with: node-version: '18' - run: yarn install --immutable --immutable-cache --check-cache diff --git a/.github/workflows/image-agent.yaml b/.github/workflows/image-agent.yaml index 0791b7d5..292c7ad3 100644 --- a/.github/workflows/image-agent.yaml +++ b/.github/workflows/image-agent.yaml @@ -17,9 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v4 - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: submodules: 'true' - name: Get tag @@ -27,13 +27,13 @@ jobs: echo "::set-output name=commit::$(git rev-parse --short HEAD)" id: get_tag - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Build, tag, push id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v7 with: context: . file: images/agent/Dockerfile diff --git a/.github/workflows/image-bchn.yaml b/.github/workflows/image-bchn.yaml index 625d545d..ceb80bf5 100644 --- a/.github/workflows/image-bchn.yaml +++ b/.github/workflows/image-bchn.yaml @@ -12,21 +12,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v4 - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Get version run: | echo "::set-output name=version::$(grep -F 'ENV BITCOIN_VERSION' images/bitcoin-cash-node/Dockerfile | grep -oE '[0-9.]+')" id: get_version - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Build, tag, push id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v7 with: context: images/bitcoin-cash-node push: true diff --git a/.github/workflows/image-hasura.yaml b/.github/workflows/image-hasura.yaml index 5a9c34a4..f61b08fd 100644 --- a/.github/workflows/image-hasura.yaml +++ b/.github/workflows/image-hasura.yaml @@ -12,21 +12,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v4 - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Get tag run: | echo "::set-output name=commit::$(git rev-parse --short HEAD)" id: get_tag - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Build, tag, push id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v7 with: context: images/hasura push: true diff --git a/.github/workflows/release-chart.yaml b/.github/workflows/release-chart.yaml index e07c20fe..27dfbd4d 100644 --- a/.github/workflows/release-chart.yaml +++ b/.github/workflows/release-chart.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: fetch-depth: 0 - name: Configure Git @@ -18,7 +18,7 @@ jobs: git config user.name "$GITHUB_ACTOR" git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - name: Install Helm - uses: azure/setup-helm@v3 + uses: azure/setup-helm@v5 - name: Run chart-releaser uses: helm/chart-releaser-action@v1.5.0 with: From cd4d0e563d072aab291751881115dfb612dd47e3 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 19 Oct 2026 11:41:14 +0100 Subject: [PATCH 50/63] style(e2e): format the re-org test Co-Authored-By: Claude Opus 5.5 --- src/e2e/e2e.spec.ts | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 1d68404f..e4fe5505 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -2023,15 +2023,13 @@ test.serial('[e2e] handles re-org of a single block', async (t) => { t.pass(); }); -test.serial( - '[e2e] new block saved after reorg', - async (t) => { - const acceptedBlocks = ( - await client.query<{ - hash: string; - nodeName: string; - }>( - /* sql */ ` +test.serial('[e2e] new block saved after reorg', async (t) => { + const acceptedBlocks = ( + await client.query<{ + hash: string; + nodeName: string; + }>( + /* sql */ ` SELECT node.name AS "nodeName", encode(block.hash, 'hex') AS hash FROM node_block INNER JOIN node @@ -2042,14 +2040,13 @@ test.serial( AND block.height = $1 ORDER BY block.hash; `, - [splitHeight + 1] - ) - ).rows; - t.deepEqual(acceptedBlocks, [ - { hash: tipA[0]!.header.hash, nodeName: 'node3' }, - ]); - } -); + [splitHeight + 1] + ) + ).rows; + t.deepEqual(acceptedBlocks, [ + { hash: tipA[0]!.header.hash, nodeName: 'node3' }, + ]); +}); test.serial('[e2e] handles reversal of single-block re-org', async (t) => { const tipStartIndex = 2; From 8625fcee5269bd97e3752118d61b4c53a514c279 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 19 Oct 2026 13:10:12 +0100 Subject: [PATCH 51/63] test(e2e): check reorg block acceptance in the database, not the save log line When two mock nodes announce the same block, the "Saved new block" line lists only the nodes whose headers arrived before the save; a later announcer is recorded via "accepted existing blocks" and never appears in a save line. Header order across separate peer connections is not deterministic, so waiting for "nodes: node1, node3" (and similar) can time out even though the agent behaved correctly. Poll node_block for the expected nodes instead, for each multi-node announcement in the reorg sequence (heights 3001, 3002, 3006, 3007, 3008). Co-Authored-By: Claude Opus 5.5 --- src/e2e/e2e.spec.ts | 118 ++++++++++++++++++++++++++++++++------------ 1 file changed, 86 insertions(+), 32 deletions(-) diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 1d68404f..21af4077 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -650,6 +650,51 @@ const waitForBlockTransactionCount = async ( ); }; +/** + * When several mock nodes announce the same block, the agent's "Saved new + * block" log line lists only the nodes whose headers arrived before the save; + * later announcers are recorded via "accepted existing blocks" instead. Header + * arrival order across peer connections isn't deterministic, so tests check + * the resulting node_block rows rather than the log line. + */ +const blockAcceptancePollingAttempts = 100; +const blockAcceptancePollingIntervalMs = 100; +const getAcceptingNodeNames = async (blockHash: string) => + ( + await client.query<{ nodeName: string }>( + /* sql */ ` + SELECT node.name AS "nodeName" + FROM node_block + INNER JOIN node ON node.internal_id = node_block.node_internal_id + INNER JOIN block ON block.internal_id = node_block.block_internal_id + WHERE block.hash = $1 + ORDER BY node.name; + `, + [hexToBin(blockHash)] + ) + ).rows.map((row) => row.nodeName); +const waitForBlockAcceptedByNodes = async ( + blockHash: string, + expectedNodeNames: string[], + remainingAttempts = blockAcceptancePollingAttempts +): Promise => { + const acceptingNodeNames = await getAcceptingNodeNames(blockHash); + if ( + expectedNodeNames.every((nodeName) => + acceptingNodeNames.includes(nodeName) + ) || + remainingAttempts === 0 + ) { + return acceptingNodeNames; + } + await sleep(blockAcceptancePollingIntervalMs); + return waitForBlockAcceptedByNodes( + blockHash, + expectedNodeNames, + remainingAttempts - 1 + ); +}; + const mempoolExpirationPollingAttempts = 50; const mempoolExpirationPollingIntervalMs = 100; const getExpiredMempoolArchiveState = async () => @@ -1995,10 +2040,13 @@ test.serial( ), '3 of 4 transactions should be new in tip A block 3001. Has the mockchain changed? (If so, update this test.)' ); - await waitForStdout( - /Saved new block – height:\s+3001[^\n]+nodes: node2, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipB[0]!.header.hash, [ + 'node2', + 'node3', + ]), + ['node2', 'node3'] ); - t.pass(); } ); @@ -2017,21 +2065,19 @@ test.serial('[e2e] handles re-org of a single block', async (t) => { await waitForStdout( /Saved new block – height:\s+3002[^\n]+hash: a44a664d5acc560305fceb9ba3c7f195a5d78236c1705d5ae7434ba784005689[^\n]+nodes: node2/u ); - await waitForStdout( - /Saved new block – height:\s+3002[^\n]+hash: 9c4feec6f35a54f2244f2ab14e1370e60713097be2ead8402e4ef68f96e07c8c[^\n]+nodes: node1, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipA[1]!.header.hash, ['node1', 'node3']), + ['node1', 'node3'] ); - t.pass(); }); -test.serial( - '[e2e] new block saved after reorg', - async (t) => { - const acceptedBlocks = ( - await client.query<{ - hash: string; - nodeName: string; - }>( - /* sql */ ` +test.serial('[e2e] new block saved after reorg', async (t) => { + const acceptedBlocks = ( + await client.query<{ + hash: string; + nodeName: string; + }>( + /* sql */ ` SELECT node.name AS "nodeName", encode(block.hash, 'hex') AS hash FROM node_block INNER JOIN node @@ -2042,14 +2088,13 @@ test.serial( AND block.height = $1 ORDER BY block.hash; `, - [splitHeight + 1] - ) - ).rows; - t.deepEqual(acceptedBlocks, [ - { hash: tipA[0]!.header.hash, nodeName: 'node3' }, - ]); - } -); + [splitHeight + 1] + ) + ).rows; + t.deepEqual(acceptedBlocks, [ + { hash: tipA[0]!.header.hash, nodeName: 'node3' }, + ]); +}); test.serial('[e2e] handles reversal of single-block re-org', async (t) => { const tipStartIndex = 2; @@ -2066,10 +2111,13 @@ test.serial('[e2e] handles reversal of single-block re-org', async (t) => { /node3: re-organization detected beginning at height: 3001. The following stale blocks were removed:/u ); await waitForStdout(/Saved new block – height:\s+3006[^\n]+nodes: node1/u); - await waitForStdout( - /Saved new block – height:\s+3006[^\n]+nodes: node2, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipB[tipEnd - 1]!.header.hash, [ + 'node2', + 'node3', + ]), + ['node2', 'node3'] ); - t.pass(); }); test.serial('[e2e] handles re-org of 6 blocks', async (t) => { @@ -2085,10 +2133,13 @@ test.serial('[e2e] handles re-org of 6 blocks', async (t) => { await waitForStdout( /node3: re-organization detected beginning at height: 3001. The following stale blocks were removed:/u ); - await waitForStdout( - /Saved new block – height:\s+3007[^\n]+nodes: node1, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipA[tipEnd - 1]!.header.hash, [ + 'node1', + 'node3', + ]), + ['node1', 'node3'] ); - t.pass(); }); test.serial('[e2e] handles reversal of 6 block re-org', async (t) => { @@ -2104,10 +2155,13 @@ test.serial('[e2e] handles reversal of 6 block re-org', async (t) => { await waitForStdout( /node3: re-organization detected beginning at height: 3001. The following stale blocks were removed:/u ); - await waitForStdout( - /Saved new block – height:\s+3008[^\n]+nodes: node2, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipB[tipEnd - 1]!.header.hash, [ + 'node2', + 'node3', + ]), + ['node2', 'node3'] ); - t.pass(); }); /** From 0ded0820a0737adb71931f7ba71337557f132980 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 19 Oct 2026 16:20:23 +0100 Subject: [PATCH 52/63] fix(db): pass node ids to the expiry statement as integer[] node.internal_id and node_transaction.node_internal_id are integer, so a node id above 32767 overflowed the smallint[] parameter and failed the whole expiry batch. Node registration consumes sequence values even for known nodes, so such ids are routine. The expiry test schemas now use the production column types and the concurrency fixture uses node ids above the smallint range. Co-Authored-By: Claude Opus 5.5 --- src/db.ts | 2 +- src/e2e/mempool-cleanup.spec.ts | 8 ++++---- src/e2e/mempool-expiry-concurrency.spec.ts | 12 +++++++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/db.ts b/src/db.ts index 82f9b8e6..891e8c89 100644 --- a/src/db.ts +++ b/src/db.ts @@ -500,7 +500,7 @@ WITH serialized AS ( expiry AS ( SELECT node_internal_id, transaction_internal_id, replaced_at FROM serialized - CROSS JOIN unnest($1::smallint[], $2::bigint[], $3::timestamp[]) + CROSS JOIN unnest($1::integer[], $2::bigint[], $3::timestamp[]) AS expiry (node_internal_id, transaction_internal_id, replaced_at) ), locked AS ( diff --git a/src/e2e/mempool-cleanup.spec.ts b/src/e2e/mempool-cleanup.spec.ts index b26dd39d..90215876 100644 --- a/src/e2e/mempool-cleanup.spec.ts +++ b/src/e2e/mempool-cleanup.spec.ts @@ -25,7 +25,7 @@ test.beforeEach(async (t) => { BEGIN; SET LOCAL search_path = pg_temp; SET LOCAL statement_timeout = '5s'; -CREATE TEMP TABLE node (internal_id smallint PRIMARY KEY, name text); +CREATE TEMP TABLE node (internal_id integer PRIMARY KEY, name text); CREATE TEMP TABLE transaction (internal_id bigint PRIMARY KEY, hash bytea UNIQUE); CREATE TEMP TABLE output (transaction_hash bytea, output_index integer, PRIMARY KEY (transaction_hash, output_index)); @@ -33,16 +33,16 @@ CREATE TEMP TABLE input (transaction_internal_id bigint, input_index integer, outpoint_transaction_hash bytea, outpoint_index integer, PRIMARY KEY (transaction_internal_id, input_index)); CREATE INDEX input_outpoint ON input (outpoint_transaction_hash, outpoint_index); -CREATE TEMP TABLE node_transaction (node_internal_id smallint, +CREATE TEMP TABLE node_transaction (node_internal_id integer, transaction_internal_id bigint, validated_at timestamp DEFAULT '2026-01-01', PRIMARY KEY (node_internal_id, transaction_internal_id)); -CREATE TEMP TABLE node_transaction_history (node_internal_id smallint, +CREATE TEMP TABLE node_transaction_history (node_internal_id integer, transaction_internal_id bigint, validated_at timestamp, replaced_at timestamp, PRIMARY KEY (node_internal_id, transaction_internal_id, validated_at)); CREATE TEMP TABLE block_transaction (block_internal_id bigint, transaction_internal_id bigint, PRIMARY KEY (transaction_internal_id, block_internal_id)); -CREATE TEMP TABLE node_block (node_internal_id smallint, +CREATE TEMP TABLE node_block (node_internal_id integer, block_internal_id bigint, accepted_at timestamp NOT NULL, PRIMARY KEY (node_internal_id, block_internal_id)); INSERT INTO node VALUES (1, 'alpha'), (2, 'beta'); diff --git a/src/e2e/mempool-expiry-concurrency.spec.ts b/src/e2e/mempool-expiry-concurrency.spec.ts index 509a2fd0..7f1e7966 100644 --- a/src/e2e/mempool-expiry-concurrency.spec.ts +++ b/src/e2e/mempool-expiry-concurrency.spec.ts @@ -37,7 +37,7 @@ test.before(async (t) => { DROP SCHEMA IF EXISTS ${schema} CASCADE; CREATE SCHEMA ${schema}; SET search_path = ${schema}; -CREATE TABLE node (internal_id smallint PRIMARY KEY, name text); +CREATE TABLE node (internal_id integer PRIMARY KEY, name text); CREATE TABLE transaction (internal_id bigint PRIMARY KEY, hash bytea UNIQUE); CREATE TABLE output (transaction_hash bytea, output_index integer, PRIMARY KEY (transaction_hash, output_index)); @@ -45,16 +45,16 @@ CREATE TABLE input (transaction_internal_id bigint, input_index integer, outpoint_transaction_hash bytea, outpoint_index integer, PRIMARY KEY (transaction_internal_id, input_index)); CREATE INDEX input_outpoint ON input (outpoint_transaction_hash, outpoint_index); -CREATE TABLE node_transaction (node_internal_id smallint, +CREATE TABLE node_transaction (node_internal_id integer, transaction_internal_id bigint, validated_at timestamp NOT NULL, PRIMARY KEY (node_internal_id, transaction_internal_id)); -CREATE TABLE node_transaction_history (node_internal_id smallint, +CREATE TABLE node_transaction_history (node_internal_id integer, transaction_internal_id bigint, validated_at timestamp, replaced_at timestamp, PRIMARY KEY (node_internal_id, transaction_internal_id, validated_at)); CREATE TABLE block_transaction (block_internal_id bigint, transaction_internal_id bigint, PRIMARY KEY (transaction_internal_id, block_internal_id)); -CREATE TABLE node_block (node_internal_id smallint, +CREATE TABLE node_block (node_internal_id integer, block_internal_id bigint, accepted_at timestamp NOT NULL, PRIMARY KEY (node_internal_id, block_internal_id)); ${cascadeFunction} @@ -82,7 +82,9 @@ const createStaleMempool = async (db: typeof database) => { await db.pool.query(/* sql */ ` TRUNCATE node, transaction, output, input, node_transaction, node_transaction_history, block_transaction, node_block; -INSERT INTO node VALUES (1, 'alpha'), (2, 'beta'); +-- production node ids are integers; node registration consumes sequence +-- values even for known nodes, so ids above the smallint range are routine +INSERT INTO node VALUES (40001, 'alpha'), (40002, 'beta'); INSERT INTO transaction SELECT id, sha256(int8send(id)) FROM generate_series(1, ${chains * depth}) id; INSERT INTO output SELECT hash, o FROM transaction, generate_series(0, 1) o; From 6b988ff8c72679b44098f90f383d9d9b24cf3c24 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 19 Oct 2026 16:20:51 +0100 Subject: [PATCH 53/63] fix(db): archive expired descendants with the earliest ancestor expiry or their block time The expiry statement deleted only the expired rows and left descendants to the history trigger's cascade, which loses two things: - the trigger only reaches descendants still in node_transaction, so a descendant expired in the same batch kept its own, later expiry (and passed it on), instead of its ancestor's earlier one; - the trigger does not check blocks, so a descendant also replaced by a block accepted by the node was archived with its ancestor's expiry instead of the block's accepted_at, and the accepted-block archive could no longer correct it. The statement now walks the same-node descendants itself: each gets the earliest expiry among its archived ancestors, or, if resolved by a block accepted by the node, the replaced_at the accepted-block archive would give it (NULL ends the cascade there). All rows are locked in (node, transaction) order and the trigger finds nothing left to cascade. Only expired rows without an expired parent expiring no later seed the walk, so an expired chain is walked once. The expiry queue takes the earliest expiries first, so an ancestor is never in a later batch than its descendant. Co-Authored-By: Claude Opus 5.5 --- src/db.ts | 187 ++++++++++++++++----- src/e2e/mempool-cleanup.spec.ts | 88 ++++++++++ src/e2e/mempool-expiry-concurrency.spec.ts | 41 ++++- 3 files changed, 266 insertions(+), 50 deletions(-) diff --git a/src/db.ts b/src/db.ts index 891e8c89..bf54161e 100644 --- a/src/db.ts +++ b/src/db.ts @@ -448,27 +448,77 @@ const expiryBatchSize = 500; const expiryAdvisoryLockKey = 1970172700; /** - * Archive expired node_transaction rows in one statement. Existing history - * triggers handle any same-node descendants: those not in the batch get the - * replaced_at of their archived ancestor. + * Resolution of mempool transaction `transaction` for node `node` by the blocks + * that node accepted, as recorded by + * {@link archiveMempoolTransactionsAcceptedByBlocks}: `accepted` if the + * transaction is in one of them, otherwise `replaced_at` is the earliest + * accepted_at of a block spending one of its outpoints (NULL if none). + */ +const acceptedBlockResolution = ( + node: string, + transaction: string +) => /* sql */ ` + SELECT EXISTS ( + SELECT 1 + FROM block_transaction + JOIN node_block + ON node_block.node_internal_id = ${node} + AND node_block.block_internal_id = block_transaction.block_internal_id + WHERE block_transaction.transaction_internal_id = ${transaction} + ) AS accepted, + (SELECT MIN(node_block.accepted_at) + FROM input mempool_input + JOIN input accepted_input + ON accepted_input.outpoint_transaction_hash = mempool_input.outpoint_transaction_hash + AND accepted_input.outpoint_index = mempool_input.outpoint_index + AND accepted_input.transaction_internal_id != ${transaction} + JOIN block_transaction + ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id + JOIN node_block + ON node_block.node_internal_id = ${node} + AND node_block.block_internal_id = block_transaction.block_internal_id + WHERE mempool_input.transaction_internal_id = ${transaction} + AND mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea + ) AS replaced_at`; + +/** + * Archive expired node_transaction rows, and their same-node descendants, in + * one statement. + * + * An expired row is left in place (not returned) if the transaction, or a + * conflicting spend of one of its outpoints, is already in a block accepted by + * the same node: {@link archiveMempoolTransactionsAcceptedByBlocks} archives + * those with the correct replaced_at (NULL or the block's accepted_at). The + * guard is part of the statement that locks and deletes the row, so it is + * evaluated against the row being archived, not a read taken earlier: a row + * inserted after the agent's accepted-block archive but before its expiry scan + * would otherwise be archived as expired/replaced even though it is confirmed. + * (Such a row is still archived if it descends from another row archived + * here, with the replaced_at described below.) * - * A row is left in place (not returned) if the transaction, or a conflicting - * spend of one of its outpoints, is already in a block accepted by the same - * node: {@link archiveMempoolTransactionsAcceptedByBlocks} archives those with - * the correct replaced_at (NULL or the block's accepted_at). The guard is part - * of the statement that locks and deletes the row, so it is evaluated against - * the row being archived, not a read taken earlier: a row inserted after the - * agent's accepted-block archive but before its expiry scan would otherwise be - * archived as expired/replaced even though it is confirmed. + * Descendants are archived here rather than by the history trigger's cascade + * (which then finds none left), for two reasons: + * - a descendant gets the earliest expiry among its archived ancestors, also + * when it is itself in the batch with a later expiry (the trigger only + * reaches descendants still in node_transaction, so it cannot carry an + * ancestor's expiry through a descendant this statement deletes itself); + * - a descendant resolved by a block accepted by the node gets the + * replaced_at the accepted-block archive would give it (NULL if confirmed, + * ending the cascade there; otherwise the block's accepted_at, passed on to + * its own descendants) instead of its ancestor's expiry. * - * Deadlocks: archiving a mempool transaction also archives its descendants - * (the history trigger's recursive cascade), so concurrent single-row expiries - * of a parent and of its descendants lock the same rows in opposite orders - * (each locks its own row, then the cascade waits for the other's) and - * deadlock; a restarted agent's stale mempool, expiring all at once on every - * pool connection, live-locked this way. Expiry therefore runs one batch at a - * time (see {@link expireMempoolTransaction}), under a transaction-level - * advisory lock (also across agents sharing the database), locks its rows in (node, transaction) order before deleting + * Only expired rows without an expired parent expiring no later seed the + * cascade (the others are reached from that parent with an earlier or equal + * replaced_at), so an expired chain is walked once rather than once per row. + * + * Deadlocks: archiving a mempool transaction also archives its descendants, so + * concurrent single-row expiries of a parent and of its descendants lock the + * same rows in opposite orders (each locks its own row, then the cascade waits + * for the other's) and deadlock; a restarted agent's stale mempool, expiring + * all at once on every pool connection, live-locked this way. Expiry therefore + * runs one batch at a time (see {@link expireMempoolTransaction}), under a + * transaction-level advisory lock (also across agents sharing the database), + * locks all rows it archives in (node, transaction) order before deleting * them, and retries a deadlock or serialization failure against other writers * (block acceptance, double-spend replacement) with backoff. */ @@ -492,7 +542,7 @@ export const archiveMempoolTransactions = async ( transactionInternalId: string; }>( /* sql */ ` -WITH serialized AS ( +WITH RECURSIVE serialized AS ( -- volatile, so materialized: scanned (and the lock taken) before any -- joined row can be produced, i.e. before any row lock below SELECT pg_advisory_xact_lock($4::bigint) @@ -503,37 +553,77 @@ expiry AS ( CROSS JOIN unnest($1::integer[], $2::bigint[], $3::timestamp[]) AS expiry (node_internal_id, transaction_internal_id, replaced_at) ), -locked AS ( - SELECT node_transaction.node_internal_id, - node_transaction.transaction_internal_id, +expired AS ( + SELECT expiry.node_internal_id, + expiry.transaction_internal_id, expiry.replaced_at FROM expiry JOIN node_transaction ON node_transaction.node_internal_id = expiry.node_internal_id AND node_transaction.transaction_internal_id = expiry.transaction_internal_id + CROSS JOIN LATERAL (${acceptedBlockResolution( + 'expiry.node_internal_id', + 'expiry.transaction_internal_id' + )} + ) resolution + WHERE NOT resolution.accepted + AND resolution.replaced_at IS NULL +), +invalidated (node_internal_id, transaction_internal_id, replaced_at) AS ( + SELECT node_internal_id, transaction_internal_id, replaced_at + FROM expired WHERE NOT EXISTS ( SELECT 1 - FROM block_transaction - JOIN node_block - ON node_block.node_internal_id = expiry.node_internal_id - AND node_block.block_internal_id = block_transaction.block_internal_id - WHERE block_transaction.transaction_internal_id = expiry.transaction_internal_id - ) - AND NOT EXISTS ( - SELECT 1 - FROM input mempool_input - JOIN input accepted_input - ON accepted_input.outpoint_transaction_hash = mempool_input.outpoint_transaction_hash - AND accepted_input.outpoint_index = mempool_input.outpoint_index - AND accepted_input.transaction_internal_id != expiry.transaction_internal_id - JOIN block_transaction - ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id - JOIN node_block - ON node_block.node_internal_id = expiry.node_internal_id - AND node_block.block_internal_id = block_transaction.block_internal_id - WHERE mempool_input.transaction_internal_id = expiry.transaction_internal_id - AND mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea + FROM input + JOIN transaction parent_transaction + ON parent_transaction.hash = input.outpoint_transaction_hash + JOIN expired expired_parent + ON expired_parent.node_internal_id = expired.node_internal_id + AND expired_parent.transaction_internal_id = parent_transaction.internal_id + AND expired_parent.replaced_at <= expired.replaced_at + WHERE input.transaction_internal_id = expired.transaction_internal_id ) + UNION + SELECT child.node_internal_id, + child.transaction_internal_id, + CASE + WHEN resolution.accepted THEN NULL::timestamp without time zone + ELSE COALESCE(resolution.replaced_at, parent.replaced_at) + END + FROM invalidated parent + JOIN transaction parent_transaction + ON parent_transaction.internal_id = parent.transaction_internal_id + JOIN input child_input + ON child_input.outpoint_transaction_hash = parent_transaction.hash + JOIN node_transaction child + ON child.node_internal_id = parent.node_internal_id + AND child.transaction_internal_id = child_input.transaction_internal_id + CROSS JOIN LATERAL (${acceptedBlockResolution( + 'child.node_internal_id', + 'child.transaction_internal_id' + )} + ) resolution + WHERE parent.replaced_at IS NOT NULL +), +archive_rows AS ( + -- confirmation takes precedence, then the earliest invalidation + SELECT node_internal_id, + transaction_internal_id, + CASE + WHEN bool_or(replaced_at IS NULL) THEN NULL::timestamp without time zone + ELSE MIN(replaced_at) + END AS replaced_at + FROM invalidated + GROUP BY node_internal_id, transaction_internal_id +), +locked AS ( + SELECT node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + archive_rows.replaced_at + FROM archive_rows + JOIN node_transaction + ON node_transaction.node_internal_id = archive_rows.node_internal_id + AND node_transaction.transaction_internal_id = archive_rows.transaction_internal_id ORDER BY node_transaction.node_internal_id, node_transaction.transaction_internal_id FOR UPDATE OF node_transaction ), @@ -614,7 +704,16 @@ const pendingExpiries = new Map< let expiryDrain: Promise | undefined; const drainExpiries = async (): Promise => { - const batch = [...pendingExpiries.entries()].slice(0, expiryBatchSize); + /* + * Earliest first: an ancestor's expiry is never in a later batch than its + * descendant's (which would then be archived with its own, later expiry). + */ + const batch = [...pendingExpiries.entries()] + .sort( + ([, a], [, b]) => + a.expiry.replacedAt.getTime() - b.expiry.replacedAt.getTime() + ) + .slice(0, expiryBatchSize); if (batch.length === 0) { return undefined; } diff --git a/src/e2e/mempool-cleanup.spec.ts b/src/e2e/mempool-cleanup.spec.ts index 90215876..48233af2 100644 --- a/src/e2e/mempool-cleanup.spec.ts +++ b/src/e2e/mempool-cleanup.spec.ts @@ -331,3 +331,91 @@ INSERT INTO node_transaction (node_internal_id, transaction_internal_id) ]); } ); + +/* + * Regression: a batch holding an expired parent and an expired descendant + * deleted both directly, so the descendant (and, via the history trigger, + * everything below it) recorded its own, later expiry instead of the + * parent's earlier one. + */ +test.serial( + '[e2e] a batched expiry records the earliest ancestor expiry for every descendant', + async (t) => { + const { pool, archiveMempoolTransactions } = t.context.db; + // chain 1 <- 2 <- 3 <- 4; only 1 and 3 are in the batch + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2 + UNION ALL SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 3; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + SELECT 1, tx FROM generate_series(1, 4) tx; +`); + const archived = await archiveMempoolTransactions([ + { + nodeInternalId: 1, + replacedAt: new Date('2026-01-01T02:00:00.000Z'), + transactionInternalId: 3, + }, + { + nodeInternalId: 1, + replacedAt: new Date('2026-01-01T01:00:00.000Z'), + transactionInternalId: 1, + }, + ]); + t.deepEqual( + archived.map((expiry) => ({ tx: expiry.transactionInternalId })), + [{ tx: 1 }, { tx: 3 }] + ); + t.deepEqual((await pool.query(membershipQuery)).rows, []); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 1, replaced: '2026-01-01 01:00:00', tx: 2 }, + { node: 1, replaced: '2026-01-01 01:00:00', tx: 3 }, + { node: 1, replaced: '2026-01-01 01:00:00', tx: 4 }, + ]); + } +); + +/* + * Regression: the block guard only covered the expired rows themselves. The + * history trigger then archived a descendant also replaced by a block accepted + * by the node (and the descendant's own descendants) with the parent's expiry, + * so the accepted-block archive could no longer record the block's accepted_at. + */ +test.serial( + '[e2e] expiry archives a descendant replaced by an accepted block with the block time', + async (t) => { + const { pool, archiveMempoolTransactions } = t.context.db; + // 2 spends 1 and 10; 5 (mined for alpha only) also spends 10; 3 spends 2 + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 2, 1, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO block_transaction VALUES (1, 5); +INSERT INTO node_block VALUES (1, 1, '2026-01-03'); +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + SELECT node, tx FROM generate_series(1, 2) node, generate_series(1, 3) tx; +`); + const replacedAt = new Date('2026-01-01T01:00:00.000Z'); + const archived = await archiveMempoolTransactions([ + { nodeInternalId: 1, replacedAt, transactionInternalId: 1 }, + { nodeInternalId: 2, replacedAt, transactionInternalId: 1 }, + ]); + t.deepEqual( + archived.map((expiry) => ({ node: expiry.nodeInternalId })), + [{ node: 1 }, { node: 2 }] + ); + t.deepEqual((await pool.query(membershipQuery)).rows, []); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 1, replaced: '2026-01-03 00:00:00', tx: 2 }, + { node: 1, replaced: '2026-01-03 00:00:00', tx: 3 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 2 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 3 }, + ]); + } +); diff --git a/src/e2e/mempool-expiry-concurrency.spec.ts b/src/e2e/mempool-expiry-concurrency.spec.ts index 7f1e7966..cd880768 100644 --- a/src/e2e/mempool-expiry-concurrency.spec.ts +++ b/src/e2e/mempool-expiry-concurrency.spec.ts @@ -78,7 +78,10 @@ const depth = 40; * transaction at the same depth, all validated at the same moment for two * nodes: a restarted agent's stale mempool, all expiring at once. */ -const createStaleMempool = async (db: typeof database) => { +const createStaleMempool = async ( + db: typeof database, + validatedAt = `'2026-01-01'::timestamp` +) => { await db.pool.query(/* sql */ ` TRUNCATE node, transaction, output, input, node_transaction, node_transaction_history, block_transaction, node_block; @@ -102,7 +105,7 @@ INSERT INTO input }) d JOIN transaction parent ON parent.internal_id = (c - 1) * ${depth} + d + 1; INSERT INTO node_transaction - SELECT node.internal_id, transaction.internal_id, '2026-01-01' + SELECT node.internal_id, transaction.internal_id, ${validatedAt} FROM node CROSS JOIN transaction; `); }; @@ -136,7 +139,8 @@ const shuffled = (items: T[], seed: number) => { const expectAllArchived = async ( t: ExecutionContext<{ db: typeof database; schema: string }>, - errors: unknown[] + errors: unknown[], + replaced = '2026-01-01 01:00:00' ) => { t.deepEqual( errors.map((err) => (err as { code?: string }).code ?? err), @@ -150,11 +154,11 @@ const expectAllArchived = async ( const replacedAt = await t.context.db.pool.query<{ replaced: string }>( /* sql */ `SELECT DISTINCT replaced_at::text AS replaced FROM node_transaction_history;` ); - t.deepEqual(replacedAt.rows, [{ replaced: '2026-01-01 01:00:00' }]); + t.deepEqual(replacedAt.rows, [{ replaced }]); }; -const staleExpiries = async (db: typeof database) => { - await createStaleMempool(db); +const staleExpiries = async (db: typeof database, validatedAt?: string) => { + await createStaleMempool(db, validatedAt); const expiring = await db.getMempoolTransactionsExpiringBefore({ expirationMs: 3_600_000, expiresBefore: new Date('2026-01-10T00:00:00.000Z'), @@ -240,3 +244,28 @@ test.serial( t.false(await db.expireMempoolTransaction(expiries[0]!)); } ); + +/* + * Every transaction descends from transaction 1, the first validated, so every + * row must record its expiry, also when a descendant's expiry is queued before + * its ancestors' and the queue needs several batches. + */ +test.serial( + '[e2e] the expiry queue archives every descendant with its earliest ancestor expiry', + async (t) => { + const { db } = t.context; + const expiries = await staleExpiries( + db, + `'2026-01-01'::timestamp + transaction.internal_id * interval '1 second'` + ); + const errors: unknown[] = []; + await Promise.all( + expiries.map(async (expiry) => + db.expireMempoolTransaction(expiry).catch((err: unknown) => { + errors.push(err); + }) + ) + ); + await expectAllArchived(t, errors, '2026-01-01 01:00:01'); + } +); From a21af66573aa5b61f2c486cd7df78969393ab8a0 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 19 Oct 2026 23:08:51 +0100 Subject: [PATCH 54/63] fix(db): treat a conflict in a block with a NULL accepted_at as a replacement in expiry Blocks accepted before Chaingraph began monitoring have a NULL accepted_at, which MIN(accepted_at) could not tell apart from no conflicting block at all. Expiry then archived a root replaced by such a block with its expiry, and a descendant replaced by one with its ancestor's expiry. The accepted-block resolution now returns whether a conflicting spend exists separately from its (nullable) earliest accepted_at: such a root is left for the accepted-block archive, and such a descendant gets that archive's NULL replaced_at. The test schemas allow a NULL node_block.accepted_at, as production does. Co-Authored-By: Claude Opus 5.5 --- src/db.ts | 43 ++++++++++-------- src/e2e/mempool-cleanup.spec.ts | 52 +++++++++++++++++++++- src/e2e/mempool-expiry-concurrency.spec.ts | 2 +- 3 files changed, 76 insertions(+), 21 deletions(-) diff --git a/src/db.ts b/src/db.ts index bf54161e..a48ea020 100644 --- a/src/db.ts +++ b/src/db.ts @@ -451,22 +451,12 @@ const expiryAdvisoryLockKey = 1970172700; * Resolution of mempool transaction `transaction` for node `node` by the blocks * that node accepted, as recorded by * {@link archiveMempoolTransactionsAcceptedByBlocks}: `accepted` if the - * transaction is in one of them, otherwise `replaced_at` is the earliest - * accepted_at of a block spending one of its outpoints (NULL if none). + * transaction is in one of them; `replaced` if one of them spends one of its + * outpoints, with `replaced_at` the earliest such block's accepted_at (NULL if + * unknown: blocks accepted before Chaingraph began monitoring). */ -const acceptedBlockResolution = ( - node: string, - transaction: string -) => /* sql */ ` - SELECT EXISTS ( - SELECT 1 - FROM block_transaction - JOIN node_block - ON node_block.node_internal_id = ${node} - AND node_block.block_internal_id = block_transaction.block_internal_id - WHERE block_transaction.transaction_internal_id = ${transaction} - ) AS accepted, - (SELECT MIN(node_block.accepted_at) +const acceptedBlockResolution = (node: string, transaction: string) => { + const conflictingAcceptedSpends = /* sql */ ` FROM input mempool_input JOIN input accepted_input ON accepted_input.outpoint_transaction_hash = mempool_input.outpoint_transaction_hash @@ -478,8 +468,21 @@ const acceptedBlockResolution = ( ON node_block.node_internal_id = ${node} AND node_block.block_internal_id = block_transaction.block_internal_id WHERE mempool_input.transaction_internal_id = ${transaction} - AND mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea + AND mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea`; + return /* sql */ ` + SELECT EXISTS ( + SELECT 1 + FROM block_transaction + JOIN node_block + ON node_block.node_internal_id = ${node} + AND node_block.block_internal_id = block_transaction.block_internal_id + WHERE block_transaction.transaction_internal_id = ${transaction} + ) AS accepted, + EXISTS (SELECT 1 ${conflictingAcceptedSpends} + ) AS replaced, + (SELECT MIN(node_block.accepted_at) ${conflictingAcceptedSpends} ) AS replaced_at`; +}; /** * Archive expired node_transaction rows, and their same-node descendants, in @@ -505,7 +508,8 @@ const acceptedBlockResolution = ( * - a descendant resolved by a block accepted by the node gets the * replaced_at the accepted-block archive would give it (NULL if confirmed, * ending the cascade there; otherwise the block's accepted_at, passed on to - * its own descendants) instead of its ancestor's expiry. + * its own descendants, or NULL if that is unknown, again ending the cascade) + * instead of its ancestor's expiry. * * Only expired rows without an expired parent expiring no later seed the * cascade (the others are reached from that parent with an earlier or equal @@ -567,7 +571,7 @@ expired AS ( )} ) resolution WHERE NOT resolution.accepted - AND resolution.replaced_at IS NULL + AND NOT resolution.replaced ), invalidated (node_internal_id, transaction_internal_id, replaced_at) AS ( SELECT node_internal_id, transaction_internal_id, replaced_at @@ -588,7 +592,8 @@ invalidated (node_internal_id, transaction_internal_id, replaced_at) AS ( child.transaction_internal_id, CASE WHEN resolution.accepted THEN NULL::timestamp without time zone - ELSE COALESCE(resolution.replaced_at, parent.replaced_at) + WHEN resolution.replaced THEN resolution.replaced_at + ELSE parent.replaced_at END FROM invalidated parent JOIN transaction parent_transaction diff --git a/src/e2e/mempool-cleanup.spec.ts b/src/e2e/mempool-cleanup.spec.ts index 48233af2..98f8b687 100644 --- a/src/e2e/mempool-cleanup.spec.ts +++ b/src/e2e/mempool-cleanup.spec.ts @@ -43,7 +43,7 @@ CREATE TEMP TABLE block_transaction (block_internal_id bigint, transaction_internal_id bigint, PRIMARY KEY (transaction_internal_id, block_internal_id)); CREATE TEMP TABLE node_block (node_internal_id integer, - block_internal_id bigint, accepted_at timestamp NOT NULL, + block_internal_id bigint, accepted_at timestamp, PRIMARY KEY (node_internal_id, block_internal_id)); INSERT INTO node VALUES (1, 'alpha'), (2, 'beta'); INSERT INTO transaction @@ -419,3 +419,53 @@ INSERT INTO node_transaction (node_internal_id, transaction_internal_id) ]); } ); + +/* + * Regression: blocks accepted before Chaingraph began monitoring have a NULL + * accepted_at, which MIN(accepted_at) could not tell apart from no conflicting + * block at all. Expiry then archived a root replaced by such a block with its + * expiry, and gave such a descendant its ancestor's expiry, instead of leaving + * the root for the accepted-block archive and giving the descendant the + * accepted-block archive's NULL (which, as there, ends the cascade). + */ +test.serial( + '[e2e] expiry treats a conflict in a block with a NULL accepted_at as replaced', + async (t) => { + const { + pool, + archiveMempoolTransactions, + archiveMempoolTransactionsAcceptedByBlocks: archive, + } = t.context.db; + // 4 and 5 (mined) spend 10; chain 1 <- 2 <- 3, 2 and 6 (mined) spend 11 + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 2, 1, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 6, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO block_transaction VALUES (1, 5), (2, 6); +INSERT INTO node_block VALUES (1, 1, NULL), (1, 2, NULL); +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 1), (1, 2), (1, 3), (1, 4); +`); + const replacedAt = new Date('2026-01-01T01:00:00.000Z'); + const archived = await archiveMempoolTransactions([ + { nodeInternalId: 1, replacedAt, transactionInternalId: 1 }, + { nodeInternalId: 1, replacedAt, transactionInternalId: 4 }, + ]); + t.deepEqual( + archived.map((expiry) => ({ tx: expiry.transactionInternalId })), + [{ tx: 1 }] + ); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 1, replaced: null, tx: 2 }, + ]); + t.deepEqual(await archive(), [ + { hash: hash('04'), nodeName: 'alpha', replacedAt: null }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, [{ node: 1, tx: 3 }]); + } +); diff --git a/src/e2e/mempool-expiry-concurrency.spec.ts b/src/e2e/mempool-expiry-concurrency.spec.ts index cd880768..41fafeb7 100644 --- a/src/e2e/mempool-expiry-concurrency.spec.ts +++ b/src/e2e/mempool-expiry-concurrency.spec.ts @@ -55,7 +55,7 @@ CREATE TABLE block_transaction (block_internal_id bigint, transaction_internal_id bigint, PRIMARY KEY (transaction_internal_id, block_internal_id)); CREATE TABLE node_block (node_internal_id integer, - block_internal_id bigint, accepted_at timestamp NOT NULL, + block_internal_id bigint, accepted_at timestamp, PRIMARY KEY (node_internal_id, block_internal_id)); ${cascadeFunction} CREATE TRIGGER cascade_history AFTER INSERT ON node_transaction_history From 04da44d5dcd1fe8846b3a49387380aee08288579 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 19 Oct 2026 23:08:51 +0100 Subject: [PATCH 55/63] fix(db): take the expiry locks in separate commands before the archival statement The advisory lock was taken inside the archival statement, whose snapshot predates the wait, so the accepted-block guards missed block inclusions committed while expiry waited, and an expired parent deleted meanwhile by block acceptance (confirmed) still seeded the descendant walk: its row lock skipped the parent, but its unconfirmed child was archived with the parent's expiry. Expiry now runs as one transaction: take the advisory lock, lock the expired rows, then archive in a separate statement, whose READ COMMITTED snapshot is taken after both waits. The cleanup spec discards its temporary schema after each test instead of rolling back an enclosing transaction, as expiry commits its own. Co-Authored-By: Claude Opus 5.5 --- src/db.ts | 57 ++++++--- src/e2e/mempool-cleanup.spec.ts | 12 +- src/e2e/mempool-expiry-concurrency.spec.ts | 134 ++++++++++++++++++++- 3 files changed, 180 insertions(+), 23 deletions(-) diff --git a/src/db.ts b/src/db.ts index a48ea020..c461ff59 100644 --- a/src/db.ts +++ b/src/db.ts @@ -445,7 +445,7 @@ const expiryBatchSize = 500; * Advisory lock key serializing mempool expiry across every agent sharing the * database (the agent also runs expiry batches one at a time). */ -const expiryAdvisoryLockKey = 1970172700; +export const expiryAdvisoryLockKey = 1970172700; /** * Resolution of mempool transaction `transaction` for node `node` by the blocks @@ -486,7 +486,12 @@ const acceptedBlockResolution = (node: string, transaction: string) => { /** * Archive expired node_transaction rows, and their same-node descendants, in - * one statement. + * one transaction: take the expiry advisory lock, lock the expired rows, then + * archive them in one statement. Each is a separate command, so (in READ + * COMMITTED) the archival statement's snapshot is taken after both waits: it + * sees block inclusions committed while expiry waited, and no expired row + * deleted meanwhile (e.g. confirmed by block acceptance) can seed the + * descendant walk, as rows it has locked cannot be deleted under it. * * An expired row is left in place (not returned) if the transaction, or a * conflicting spend of one of its outpoints, is already in a block accepted by @@ -522,9 +527,10 @@ const acceptedBlockResolution = (node: string, transaction: string) => { * all at once on every pool connection, live-locked this way. Expiry therefore * runs one batch at a time (see {@link expireMempoolTransaction}), under a * transaction-level advisory lock (also across agents sharing the database), - * locks all rows it archives in (node, transaction) order before deleting - * them, and retries a deadlock or serialization failure against other writers - * (block acceptance, double-spend replacement) with backoff. + * locks the rows it archives in (node, transaction) order (the expired rows, + * then their descendants) before deleting them, and retries a deadlock or + * serialization failure against other writers (block acceptance, double-spend + * replacement) with backoff. */ export const archiveMempoolTransactions = async ( expiries: MempoolTransactionExpiry[] @@ -537,24 +543,39 @@ export const archiveMempoolTransactions = async ( a.nodeInternalId - b.nodeInternalId || a.transactionInternalId - b.transactionInternalId ); + const nodeInternalIds = sorted.map((expiry) => expiry.nodeInternalId); + const transactionInternalIds = sorted.map( + (expiry) => expiry.transactionInternalId + ); const archiveOnce = async () => { const client = await pool.connect(); // eslint-disable-next-line functional/no-try-statement try { + await client.query('BEGIN;'); + await client.query('SELECT pg_advisory_xact_lock($1::bigint);', [ + expiryAdvisoryLockKey, + ]); + await client.query( + /* sql */ ` +SELECT 1 + FROM node_transaction + JOIN unnest($1::integer[], $2::bigint[]) + AS expiry (node_internal_id, transaction_internal_id) + ON node_transaction.node_internal_id = expiry.node_internal_id + AND node_transaction.transaction_internal_id = expiry.transaction_internal_id + ORDER BY node_transaction.node_internal_id, node_transaction.transaction_internal_id + FOR UPDATE OF node_transaction; +`, + [nodeInternalIds, transactionInternalIds] + ); const result = await client.query<{ nodeInternalId: string; transactionInternalId: string; }>( /* sql */ ` -WITH RECURSIVE serialized AS ( - -- volatile, so materialized: scanned (and the lock taken) before any - -- joined row can be produced, i.e. before any row lock below - SELECT pg_advisory_xact_lock($4::bigint) -), -expiry AS ( +WITH RECURSIVE expiry AS ( SELECT node_internal_id, transaction_internal_id, replaced_at - FROM serialized - CROSS JOIN unnest($1::integer[], $2::bigint[], $3::timestamp[]) + FROM unnest($1::integer[], $2::bigint[], $3::timestamp[]) AS expiry (node_internal_id, transaction_internal_id, replaced_at) ), expired AS ( @@ -653,12 +674,12 @@ SELECT node_internal_id::text AS "nodeInternalId", FROM inserted_history; `, [ - sorted.map((expiry) => expiry.nodeInternalId), - sorted.map((expiry) => expiry.transactionInternalId), + nodeInternalIds, + transactionInternalIds, sorted.map((expiry) => expiry.replacedAt.toISOString()), - expiryAdvisoryLockKey, ] ); + await client.query('COMMIT;'); const archived = new Set( result.rows.map( (row) => `${row.nodeInternalId}:${row.transactionInternalId}` @@ -667,6 +688,10 @@ SELECT node_internal_id::text AS "nodeInternalId", return sorted.filter((expiry) => archived.has(`${expiry.nodeInternalId}:${expiry.transactionInternalId}`) ); + } catch (err) { + await client.query('ROLLBACK;'); + // eslint-disable-next-line functional/no-throw-statement + throw err; } finally { client.release(); } diff --git a/src/e2e/mempool-cleanup.spec.ts b/src/e2e/mempool-cleanup.spec.ts index 98f8b687..839c7c89 100644 --- a/src/e2e/mempool-cleanup.spec.ts +++ b/src/e2e/mempool-cleanup.spec.ts @@ -11,8 +11,9 @@ const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; /* - * A single connection keeps every operation inside the same temporary schema - * and rollback transaction. No persistent tables are created or modified. + * A single connection keeps every operation inside the same temporary schema, + * discarded after each test (not a rollback transaction: expiry runs its own + * transaction). No persistent tables are created or modified. */ test.before(async (t) => { process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = `postgres://chaingraph:very_insecure_postgres_password@${host}:${port}/postgres`; @@ -22,9 +23,8 @@ test.before(async (t) => { test.beforeEach(async (t) => { await t.context.db.pool.query(/* sql */ ` -BEGIN; -SET LOCAL search_path = pg_temp; -SET LOCAL statement_timeout = '5s'; +SET search_path = pg_temp; +SET statement_timeout = '5s'; CREATE TEMP TABLE node (internal_id integer PRIMARY KEY, name text); CREATE TEMP TABLE transaction (internal_id bigint PRIMARY KEY, hash bytea UNIQUE); CREATE TEMP TABLE output (transaction_hash bytea, output_index integer, @@ -72,7 +72,7 @@ CREATE TRIGGER cascade_history AFTER INSERT ON node_transaction_history }); test.afterEach.always(async (t) => { - await t.context.db.pool.query('ROLLBACK;'); + await t.context.db.pool.query('DISCARD TEMP;'); }); test.after.always(async (t) => { diff --git a/src/e2e/mempool-expiry-concurrency.spec.ts b/src/e2e/mempool-expiry-concurrency.spec.ts index 41fafeb7..ed723558 100644 --- a/src/e2e/mempool-expiry-concurrency.spec.ts +++ b/src/e2e/mempool-expiry-concurrency.spec.ts @@ -6,7 +6,7 @@ import baseTest from 'ava'; import type * as database from '../db.js'; -// cspell:words lpad +// cspell:words lpad xact const test = baseTest as TestFn<{ db: typeof database; schema: string }>; const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; @@ -269,3 +269,135 @@ test.serial( await expectAllArchived(t, errors, '2026-01-01 01:00:01'); } ); + +/* + * Expire transaction 1 of a two-transaction mempool for alpha (2 spends 1, and + * `prepare` runs first) while another connection's transaction, `blocker`, + * holds a lock the expiry waits for. Once the expiry waits, `concurrently` is + * committed on a third connection, then `blocker`. + */ +const expireWhileBlocked = async ( + db: typeof database, + { + blocker, + concurrently = '', + prepare = '', + }: { blocker: string; concurrently?: string; prepare?: string } +) => { + await db.pool.query(/* sql */ ` +TRUNCATE node, transaction, output, input, node_transaction, + node_transaction_history, block_transaction, node_block; +INSERT INTO node VALUES (40001, 'alpha'); +INSERT INTO transaction + SELECT id, sha256(int8send(id)) FROM generate_series(1, 2) id; +INSERT INTO output SELECT hash, 0 FROM transaction; +INSERT INTO input SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1; +INSERT INTO node_transaction + SELECT 40001, internal_id, '2026-01-01' FROM transaction; +${prepare} +`); + const blocking = await db.pool.connect(); + const concurrent = await db.pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + await blocking.query(`BEGIN; ${blocker}`); + const expiry = db.archiveMempoolTransactions([ + { + nodeInternalId: 40001, + replacedAt: new Date('2026-01-01T01:00:00.000Z'), + transactionInternalId: 1, + }, + ]); + const waitForExpiryToBlock = async (): Promise => { + const waiting = await db.pool.query( + /* sql */ `SELECT 1 FROM pg_locks WHERE NOT granted;` + ); + if (waiting.rowCount === 0) { + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + return waitForExpiryToBlock(); + } + return undefined; + }; + await waitForExpiryToBlock(); + if (concurrently !== '') { + await concurrent.query(concurrently); + } + await blocking.query('COMMIT;'); + return await expiry; + } finally { + blocking.release(); + concurrent.release(); + } +}; + +const currentAndHistory = async (db: typeof database) => + ( + await db.pool.query<{ + replaced: string | null; + table: string; + tx: number; + }>(/* sql */ ` +SELECT 'current' AS table, transaction_internal_id::integer AS tx, + NULL AS replaced + FROM node_transaction +UNION ALL +SELECT 'history', transaction_internal_id::integer, replaced_at::text + FROM node_transaction_history +ORDER BY 1, 2;`) + ).rows; + +/* + * Regression: the advisory lock was taken inside the archival statement, whose + * snapshot predates the wait, so a block inclusion committed while it waited + * (here, block_transaction repaired for a block the node already accepted) + * was invisible to the accepted-block guard and the confirmed transaction was + * archived with its expiry. + */ +test.serial( + '[e2e] expiry sees a block inclusion committed while it waits for the expiry lock', + async (t) => { + const { db } = t.context; + const archived = await expireWhileBlocked(db, { + // another agent's expiry batch + blocker: `SELECT pg_advisory_xact_lock(${db.expiryAdvisoryLockKey});`, + concurrently: `INSERT INTO block_transaction VALUES (1, 1);`, + prepare: `INSERT INTO node_block VALUES (40001, 1, '2026-01-02');`, + }); + t.deepEqual(archived, []); + t.deepEqual(await currentAndHistory(db), [ + { replaced: null, table: 'current', tx: 1 }, + { replaced: null, table: 'current', tx: 2 }, + ]); + } +); + +/* + * Regression: an expiring parent confirmed concurrently (deleted by block + * acceptance, as by the node_block trigger) was skipped by the archival + * statement's row locks, but its snapshot still had the parent, so its + * unconfirmed child was archived with the parent's expiry. + */ +test.serial( + '[e2e] expiry leaves the child of a parent confirmed while the expiry waits', + async (t) => { + const { db } = t.context; + const archived = await expireWhileBlocked(db, { + blocker: `INSERT INTO block_transaction VALUES (1, 1); +INSERT INTO node_block VALUES (40001, 1, '2026-01-02'); +WITH confirmed AS ( + DELETE FROM node_transaction + WHERE node_internal_id = 40001 AND transaction_internal_id = 1 + RETURNING node_internal_id, transaction_internal_id, validated_at) +INSERT INTO node_transaction_history + SELECT node_internal_id, transaction_internal_id, validated_at, NULL + FROM confirmed;`, + }); + t.deepEqual(archived, []); + t.deepEqual(await currentAndHistory(db), [ + { replaced: null, table: 'current', tx: 2 }, + { replaced: null, table: 'history', tx: 1 }, + ]); + } +); From c3278dfeb8ef3b65f18690f569cc4d1b724b4ecf Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Mon, 19 Oct 2026 23:41:44 +0100 Subject: [PATCH 56/63] fix(db): retry expiry when a walked descendant is deleted before its lock The archival statement walks descendants in its snapshot but locks them only afterwards. For P -> C -> G with only P expiring, C confirmed concurrently (deleted with replaced_at NULL, ending the cascade) was skipped by the row locks, but the walk had already reached G through the old C, so G was archived with P's expiry. The statement now deletes nothing unless every walked row was still present when locked; otherwise the transaction is rolled back and retried with a fresh snapshot under the existing backoff. If retries are exhausted it throws, leaving the rows for the next expiration scan. Co-Authored-By: Claude Opus 5.5 --- src/db.ts | 55 +++++++++++++++------- src/e2e/mempool-expiry-concurrency.spec.ts | 38 +++++++++++++++ 2 files changed, 76 insertions(+), 17 deletions(-) diff --git a/src/db.ts b/src/db.ts index c461ff59..0c396bff 100644 --- a/src/db.ts +++ b/src/db.ts @@ -434,10 +434,14 @@ export interface MempoolTransactionExpiry { // cspell:words xact unnest const deadlockDetectedCode = '40P01'; const serializationFailureCode = '40001'; +/** A descendant walked by the archival statement was deleted before its lock. */ +const descendantVanishedCode = 'chaingraph-expiry-descendant-vanished'; const isRetriedConflict = (err: unknown) => - [deadlockDetectedCode, serializationFailureCode].includes( - (err as { code?: string } | undefined)?.code ?? '' - ); + [ + deadlockDetectedCode, + serializationFailureCode, + descendantVanishedCode, + ].includes((err as { code?: string } | undefined)?.code ?? ''); const expiryRetryLimit = 6; const expiryRetryBaseMs = 50; const expiryBatchSize = 500; @@ -492,6 +496,10 @@ const acceptedBlockResolution = (node: string, transaction: string) => { * sees block inclusions committed while expiry waited, and no expired row * deleted meanwhile (e.g. confirmed by block acceptance) can seed the * descendant walk, as rows it has locked cannot be deleted under it. + * Descendants are only locked by the archival statement itself, after its walk: + * if one was deleted meanwhile (e.g. confirmed, ending the cascade there), the + * walk may have reached its own descendants through it, so the statement + * deletes nothing and the transaction is retried with a fresh snapshot. * * An expired row is left in place (not returned) if the transaction, or a * conflicting spend of one of its outpoints, is already in a block accepted by @@ -528,9 +536,11 @@ const acceptedBlockResolution = (node: string, transaction: string) => { * runs one batch at a time (see {@link expireMempoolTransaction}), under a * transaction-level advisory lock (also across agents sharing the database), * locks the rows it archives in (node, transaction) order (the expired rows, - * then their descendants) before deleting them, and retries a deadlock or - * serialization failure against other writers (block acceptance, double-spend - * replacement) with backoff. + * then their descendants) before deleting them, and retries a deadlock, + * serialization failure or vanished descendant (see above) against other + * writers (block acceptance, double-spend replacement) with backoff; once + * retries are exhausted it throws, leaving the rows for the next expiration + * scan. */ export const archiveMempoolTransactions = async ( expiries: MempoolTransactionExpiry[] @@ -569,8 +579,8 @@ SELECT 1 [nodeInternalIds, transactionInternalIds] ); const result = await client.query<{ - nodeInternalId: string; - transactionInternalId: string; + archived: string | null; + complete: boolean; }>( /* sql */ ` WITH RECURSIVE expiry AS ( @@ -653,10 +663,15 @@ locked AS ( ORDER BY node_transaction.node_internal_id, node_transaction.transaction_internal_id FOR UPDATE OF node_transaction ), +-- false if a walked row was deleted before its lock (the walk may be stale) +complete AS ( + SELECT (SELECT count(*) FROM locked) = (SELECT count(*) FROM archive_rows) AS complete +), deleted_rows AS ( DELETE FROM node_transaction USING locked - WHERE node_transaction.node_internal_id = locked.node_internal_id + WHERE (SELECT complete FROM complete) + AND node_transaction.node_internal_id = locked.node_internal_id AND node_transaction.transaction_internal_id = locked.transaction_internal_id RETURNING node_transaction.node_internal_id, node_transaction.transaction_internal_id, @@ -669,9 +684,10 @@ inserted_history AS ( FROM deleted_rows RETURNING node_internal_id, transaction_internal_id ) -SELECT node_internal_id::text AS "nodeInternalId", - transaction_internal_id::text AS "transactionInternalId" - FROM inserted_history; +SELECT complete.complete, + inserted_history.node_internal_id || ':' || inserted_history.transaction_internal_id AS archived + FROM complete + LEFT JOIN inserted_history ON true; `, [ nodeInternalIds, @@ -679,12 +695,17 @@ SELECT node_internal_id::text AS "nodeInternalId", sorted.map((expiry) => expiry.replacedAt.toISOString()), ] ); + if (result.rows[0]?.complete !== true) { + // eslint-disable-next-line functional/no-throw-statement + throw Object.assign( + new Error( + 'Mempool expiry: a descendant was deleted while the expiry waited for its lock.' + ), + { code: descendantVanishedCode } + ); + } await client.query('COMMIT;'); - const archived = new Set( - result.rows.map( - (row) => `${row.nodeInternalId}:${row.transactionInternalId}` - ) - ); + const archived = new Set(result.rows.map((row) => row.archived)); return sorted.filter((expiry) => archived.has(`${expiry.nodeInternalId}:${expiry.transactionInternalId}`) ); diff --git a/src/e2e/mempool-expiry-concurrency.spec.ts b/src/e2e/mempool-expiry-concurrency.spec.ts index ed723558..3084452a 100644 --- a/src/e2e/mempool-expiry-concurrency.spec.ts +++ b/src/e2e/mempool-expiry-concurrency.spec.ts @@ -401,3 +401,41 @@ INSERT INTO node_transaction_history ]); } ); + +/* + * Regression: the archival statement walked descendants in a snapshot taken + * before it waited for their row locks. For P (1) → C (2) → G (3) with only P + * expiring, C confirmed concurrently (deleted with replaced_at NULL, ending + * the cascade) was skipped by the row locks, but the walk had already reached + * G through the old C, so G was archived with P's expiry. + */ +test.serial( + '[e2e] expiry leaves the grandchild of a child confirmed while the expiry waits', + async (t) => { + const { db } = t.context; + const archived = await expireWhileBlocked(db, { + blocker: `INSERT INTO block_transaction VALUES (1, 2); +INSERT INTO node_block VALUES (40001, 1, '2026-01-02'); +WITH confirmed AS ( + DELETE FROM node_transaction + WHERE node_internal_id = 40001 AND transaction_internal_id = 2 + RETURNING node_internal_id, transaction_internal_id, validated_at) +INSERT INTO node_transaction_history + SELECT node_internal_id, transaction_internal_id, validated_at, NULL + FROM confirmed;`, + prepare: `INSERT INTO transaction VALUES (3, sha256(int8send(3::bigint))); +INSERT INTO output VALUES (sha256(int8send(3::bigint)), 0); +INSERT INTO input SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO node_transaction VALUES (40001, 3, '2026-01-01');`, + }); + t.deepEqual( + archived.map((expiry) => expiry.transactionInternalId), + [1] + ); + t.deepEqual(await currentAndHistory(db), [ + { replaced: null, table: 'current', tx: 3 }, + { replaced: '2026-01-01 01:00:00', table: 'history', tx: 1 }, + { replaced: null, table: 'history', tx: 2 }, + ]); + } +); From 3764c67f9a5a9e000fc92b1b48007586b0f0dbf3 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 21 Oct 2026 06:55:46 +0100 Subject: [PATCH 57/63] fix(agent): survive Postgres restarts instead of crashing (#17) Attach pool and per-client error listeners so a Postgres restart (57P01 etc.) is logged instead of crashing the agent via uncaughtException; the incomplete-block repair scan retries after a lost connection. Block-save failures stay fatal. Adds unit tests and an e2e test that terminates every agent backend. Co-Authored-By: Claude Opus 5.5 --- src/agent.ts | 23 ++++++++ src/e2e/e2e.spec.ts | 31 +++++++++++ src/index.ts | 13 +++++ src/postgres-errors.spec.ts | 68 ++++++++++++++++++++++++ src/postgres-errors.ts | 103 ++++++++++++++++++++++++++++++++++++ 5 files changed, 238 insertions(+) create mode 100644 src/postgres-errors.spec.ts create mode 100644 src/postgres-errors.ts diff --git a/src/agent.ts b/src/agent.ts index ddd25dee..17a50fbf 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -58,6 +58,10 @@ import { saveTransactionForNodes, } from './db.js'; import type { ExpiringMempoolTransaction, IncompleteBlock } from './db.js'; +import { + describePostgresError, + isPostgresConnectionError, +} from './postgres-errors.js'; import type { ChaingraphBlock } from './types/chaingraph.js'; // eslint-disable-next-line @typescript-eslint/naming-convention @@ -116,6 +120,11 @@ const renderSyncPercentage = (value: number) => { }; const msPerSecond = 1000; +/** + * Delay before retrying an incomplete block repair scan which failed because + * the Postgres connection was lost (e.g. Postgres is restarting). + */ +const incompleteBlockRepairRetryDelayMs = 5_000; const durationDecimalPlaces = 1; const transactionRateDecimalPlaces = 2; @@ -1167,6 +1176,20 @@ export class Agent { this.incompleteBlockRepairTimeout = setTimeout(() => { this.scheduledIncompleteBlockRepair = false; this.repairIncompleteBlocks().catch((err) => { + if (isPostgresConnectionError(err) && !this.willShutdown) { + this.logger.warn( + `Agent: incomplete block repair scan lost its Postgres connection (${describePostgresError( + err + )}); retrying in ${ + incompleteBlockRepairRetryDelayMs / msPerSecond + } seconds.` + ); + this.incompleteBlockRepairTimeout = setTimeout(() => { + this.incompleteBlockRepairTimeout = undefined; + this.scheduleIncompleteBlockRepair(); + }, incompleteBlockRepairRetryDelayMs); + return; + } this.logger.fatal(err); this.shutdown().catch((shutdownErr) => { this.logger.error(shutdownErr); diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index f45883c4..d6a28207 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -2020,6 +2020,37 @@ const newBlocks = ( ); }; +test.serial( + '[e2e] survives Postgres terminating its connections (e.g. a restart)', + async (t) => { + const crashGracePeriodMs = 2_000; + const logStart = stdoutBuffer.length; + const terminated = await client.query<{ terminated: boolean }>( + /* sql */ ` + SELECT pg_terminate_backend(pid) AS terminated FROM pg_stat_activity + -- cspell:disable-next-line + WHERE datname = $1 + AND pid <> pg_backend_pid(); + `, + [e2eTestDbName] + ); + t.true( + terminated.rows.some((row) => row.terminated), + 'Expected the agent to hold at least one Postgres connection.' + ); + await waitForStdout('Postgres connection lost'); + // give a crash (uncaught 'error' event → shutdown) time to happen + await sleep(crashGracePeriodMs); + const logsSinceTermination = stdoutBuffer.slice(logStart); + t.false(logsSinceTermination.includes('uncaughtException')); + t.false(logsSinceTermination.includes('Shutting down...')); + const healthCheckResponse = await got( + `http://localhost:${chaingraphInternalApiPort}/health-check` + ); + t.deepEqual(healthCheckResponse.statusCode, StatusCode.success); + } +); + test.serial( '[e2e] syncs blocks as they arrive, handles multiple chain tips', async (t) => { diff --git a/src/index.ts b/src/index.ts index 4149d744..449ac818 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,11 +5,24 @@ import bitcoreP2pCash from '@chaingraph/bitcore-p2p-cash'; import { Agent, cancelableDelay } from './agent.js'; import { chaingraphInternalApiPort } from './config.js'; +import { pool } from './db.js'; import { instantiateLogger } from './logging.js'; +import { + describePostgresError, + handlePoolConnectionErrors, +} from './postgres-errors.js'; const logger = instantiateLogger(); logger.info(`Starting Chaingraph...`); +handlePoolConnectionErrors(pool, (err) => { + logger.warn( + `Postgres connection lost (${describePostgresError( + err + )}); discarded it, the next query opens a new connection.` + ); +}); + const shutdownTasks = [] as (() => void)[]; const agent = new Agent({ logger, diff --git a/src/postgres-errors.spec.ts b/src/postgres-errors.spec.ts new file mode 100644 index 00000000..f0cd5f64 --- /dev/null +++ b/src/postgres-errors.spec.ts @@ -0,0 +1,68 @@ +import { EventEmitter } from 'events'; + +import test from 'ava'; +import type pg from 'pg'; + +import { + describePostgresError, + handlePoolConnectionErrors, + isPostgresConnectionError, +} from './postgres-errors.js'; + +const withCode = (code: string, message = 'error') => + Object.assign(new Error(message), { code }); + +test('isPostgresConnectionError: recognizes lost connections', (t) => { + t.true( + isPostgresConnectionError( + withCode('57P01', 'terminating connection due to administrator command') + ) + ); + t.true(isPostgresConnectionError(withCode('57P03'))); + t.true(isPostgresConnectionError(withCode('08006'))); + t.true(isPostgresConnectionError(withCode('ECONNREFUSED'))); + t.true(isPostgresConnectionError(withCode('ECONNRESET'))); + t.true( + isPostgresConnectionError(new Error('Connection terminated unexpectedly')) + ); + t.true( + isPostgresConnectionError( + new Error( + 'Client has encountered a connection error and is not queryable' + ) + ) + ); +}); + +test('isPostgresConnectionError: ignores query and data errors', (t) => { + t.false(isPostgresConnectionError(withCode('23505', 'duplicate key'))); + t.false(isPostgresConnectionError(withCode('42P01', 'missing relation'))); + t.false(isPostgresConnectionError(new Error('something else'))); + t.false(isPostgresConnectionError(undefined)); + t.false(isPostgresConnectionError('Connection terminated')); +}); + +test('describePostgresError: includes the code, omits attached state', (t) => { + const err = Object.assign(withCode('57P01', 'terminating connection'), { + client: { large: 'state' }, + }); + t.is(describePostgresError(err), '57P01: terminating connection'); + t.is(describePostgresError(new Error('plain')), 'plain'); +}); + +test('handlePoolConnectionErrors: client and pool errors do not throw', (t) => { + const pool = new EventEmitter(); + const client = new EventEmitter(); + const reported: Error[] = []; + handlePoolConnectionErrors(pool as unknown as pg.Pool, (err) => { + reported.push(err); + }); + pool.emit('connect', client); + const terminated = withCode('57P01'); + t.notThrows(() => client.emit('error', terminated)); + t.notThrows(() => + client.emit('error', new Error('Connection terminated unexpectedly')) + ); + t.notThrows(() => pool.emit('error', terminated, client)); + t.deepEqual(reported, [terminated]); +}); diff --git a/src/postgres-errors.ts b/src/postgres-errors.ts new file mode 100644 index 00000000..9db449de --- /dev/null +++ b/src/postgres-errors.ts @@ -0,0 +1,103 @@ +import type pg from 'pg'; + +/** + * Error codes Postgres sends when it closes a session: `57P01` + * (admin_shutdown – fast shutdown/restart or `pg_terminate_backend`), `57P02` + * (crash_shutdown), `57P03` (cannot_connect_now – starting up or shutting + * down). + */ +const connectionLossSqlStates = ['57P01', '57P02', '57P03']; +/** + * Socket-level errors seen while Postgres is restarting or unreachable. + */ +const connectionLossSystemErrorCodes = [ + 'ECONNREFUSED', + 'ECONNRESET', + 'EPIPE', + 'ETIMEDOUT', + 'EHOSTUNREACH', + 'ENOTFOUND', + 'EAI_AGAIN', +]; +/** + * Messages node-postgres uses for connections that ended underneath a client. + */ +const connectionLossMessages = [ + 'Connection terminated', + 'Client has encountered a connection error and is not queryable', +]; + +const isConnectionLossCode = (code: unknown) => + typeof code === 'string' && + (connectionLossSqlStates.includes(code) || + code.startsWith('08') || + connectionLossSystemErrorCodes.includes(code)); + +const isConnectionLossMessage = (message: unknown) => + typeof message === 'string' && + connectionLossMessages.some((lossMessage) => message.includes(lossMessage)); + +/** + * Returns `true` if `err` indicates the Postgres connection was lost (e.g. + * Postgres restarted), rather than a problem with the query or the data. + * Operations failing with such an error can be retried once Postgres is back. + * @param err - the error to test + */ +export const isPostgresConnectionError = (err: unknown) => { + if (typeof err !== 'object' || err === null) { + return false; + } + const { code, message } = err as { code?: unknown; message?: unknown }; + return isConnectionLossCode(code) || isConnectionLossMessage(message); +}; + +/** + * Summarize a Postgres connection error for logging without the attached + * client/socket state (node-postgres errors can carry the whole client). + * @param err - the error to describe + */ +export const describePostgresError = (err: unknown) => { + if (typeof err !== 'object' || err === null) { + return String(err); + } + const { code, message } = err as { code?: unknown; message?: unknown }; + const text = typeof message === 'string' ? message : String(err); + return typeof code === 'string' ? `${code}: ${text}` : text; +}; + +/** + * Keep a lost Postgres connection from crashing the process. + * + * When Postgres restarts (or a connection is terminated), node-postgres emits + * an `error` event: on the pool for idle clients, and on the client itself for + * checked-out clients between queries. With no listener, Node.js throws it as + * an uncaught exception. The broken client is already unusable: the pool + * discards it on release (or immediately, if idle), and the next `connect()` + * opens a fresh connection, so the only thing to do here is log it. Queries in + * flight still reject and are handled by their callers. + * @param targetPool - the pool to protect + * @param onConnectionError - called once per lost connection + */ +export const handlePoolConnectionErrors = ( + targetPool: pg.Pool, + onConnectionError: (err: Error) => void +) => { + targetPool.on('connect', (client) => { + // eslint-disable-next-line functional/no-let + let reported = false; + // a terminated client can emit more than once (error, then socket end) + client.on('error', (err) => { + if (reported) { + return; + } + reported = true; + onConnectionError(err); + }); + }); + /* + * Idle clients also emit through the pool (after the client-level listener + * above has already reported the error); this listener only prevents the + * pool's re-emitted `error` from being thrown. + */ + targetPool.on('error', () => undefined); +}; From b4d10d9664ddb8cea70e3ba997c8a95d598399e6 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 21 Oct 2026 09:14:54 +0100 Subject: [PATCH 58/63] fix(agent): do not treat already-known headers as stale on deep reorgs (fixes #80) (#7) Headers the agent already knows (e.g. the shared prefix of a deep reorg's headers message) are skipped before replacing blocks and reporting stale hashes, so valid node_block rows are no longer removed on deep reorgs. Adds regression tests for chain extensions, shallow reorgs and deep reorgs. Fixes bitauth/chaingraph#80. Co-Authored-By: Claude Opus 5.5 --- src/components/block-tree.spec.ts | 87 ++++++++++++++++++++++++++++++- src/components/block-tree.ts | 20 +++++-- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/src/components/block-tree.spec.ts b/src/components/block-tree.spec.ts index 378a9dda..09ee6324 100644 --- a/src/components/block-tree.spec.ts +++ b/src/components/block-tree.spec.ts @@ -1,8 +1,10 @@ /* eslint-disable @typescript-eslint/no-magic-numbers */ -import { range } from '@bitauth/libauth'; +import { hexToBin, range } from '@bitauth/libauth'; +import type { BitcoreBlockHeader } from '@chaingraph/bitcore-p2p-cash'; import test from 'ava'; +import { pino } from 'pino'; -import { selectHeaderLocatorHashes } from './block-tree.js'; +import { BlockTree, selectHeaderLocatorHashes } from './block-tree.js'; test('selectHeaderLocatorHashes', async (t) => { const testLength = 654247; @@ -33,3 +35,84 @@ test('selectHeaderLocatorHashes: `100`', async (t) => { const expected = [99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 86, 82, 74, 58, 26, 0]; t.deepEqual(selectHeaderLocatorHashes(headerChain), expected); }); + +/** + * Build a fake header-hash chain: the hash of each block encodes its branch + * and height, e.g. `blockHash('a', 3)`. + */ +const blockHash = (branch: string, height: number) => + `${branch}${height.toString(16).padStart(63, '0')}`; + +const createHeaders = (hashes: string[], previousHash: string) => + hashes.map((hash, index) => ({ + hash, + prevHash: Buffer.from( + hexToBin(index === 0 ? previousHash : hashes[index - 1]!).reverse() + ), + })) as unknown as BitcoreBlockHeader[]; + +/** + * Create a `BlockTree` for node `n` restored to the provided chain, recording + * each `onStaleBlocks` call. + */ +const createBlockTree = (chain: string[]) => { + const staleCalls: { staleChain: string[]; firstHeight: number }[] = []; + const blockTree = new BlockTree({ + genesisBlockByNode: { n: [chain[0]!] }, + logger: pino({ level: 'silent' }), + onStaleBlocks: (staleChain, firstHeight) => { + staleCalls.push({ firstHeight, staleChain }); + }, + }); + blockTree.restoreChainForNode('n', chain); + return { blockTree, staleCalls }; +}; + +const readChain = (blockTree: BlockTree) => + range(blockTree.getBestHeights().n! + 1).map((height) => + blockTree.getBlockHeaderHash('n', height) + ); + +/** + * Genesis (height 0) followed by blocks `1` through `tip` of `branch`, with + * heights up to `forkHeight` shared with branch `a`. + */ +const branchChain = (branch: string, forkHeight: number, tip: number) => + range(tip + 1).map((height) => + blockHash(height <= forkHeight ? 'a' : branch, height) + ); + +test('BlockTree.updateHeaders: extends the chain without stale blocks', (t) => { + const { blockTree, staleCalls } = createBlockTree(branchChain('a', 10, 10)); + const newHashes = branchChain('a', 13, 13).slice(11); + blockTree.updateHeaders('n', createHeaders(newHashes, blockHash('a', 10))); + t.deepEqual(staleCalls, []); + t.deepEqual(readChain(blockTree), branchChain('a', 13, 13)); +}); + +test('BlockTree.updateHeaders: shallow re-organization starting after the fork point', (t) => { + const { blockTree, staleCalls } = createBlockTree(branchChain('a', 10, 10)); + const newHashes = branchChain('b', 8, 11).slice(9); + blockTree.updateHeaders('n', createHeaders(newHashes, blockHash('a', 8))); + t.deepEqual(staleCalls, [ + { firstHeight: 9, staleChain: [blockHash('a', 9), blockHash('a', 10)] }, + ]); + t.deepEqual(readChain(blockTree), branchChain('b', 8, 11)); +}); + +test('BlockTree.updateHeaders: deep re-organization with headers starting below the fork point only marks diverging blocks as stale', (t) => { + const { blockTree, staleCalls } = createBlockTree(branchChain('a', 105, 105)); + /** + * As when a node answers `getheaders` from the latest locator hash in its + * active chain (here genesis): heights 1-5 are shared with the known chain. + */ + const newHashes = branchChain('b', 5, 106).slice(1); + blockTree.updateHeaders('n', createHeaders(newHashes, blockHash('a', 0))); + t.deepEqual(staleCalls, [ + { + firstHeight: 6, + staleChain: range(100, 6).map((height) => blockHash('a', height)), + }, + ]); + t.deepEqual(readChain(blockTree), branchChain('b', 5, 106)); +}); diff --git a/src/components/block-tree.ts b/src/components/block-tree.ts index 602c5950..120ff6ac 100644 --- a/src/components/block-tree.ts +++ b/src/components/block-tree.ts @@ -206,16 +206,28 @@ export class BlockTree { return this.getLocatorForNode(nodeName); } const newHashes = headers.map((header) => header.hash); + /** + * A `headers` reply may begin below the fork point (e.g. when the fork + * block is not in our locator), so leading headers can match blocks we + * already have. Only headers from the first differing height replace + * (and make stale) existing blocks. + */ + const firstDivergingIndex = newHashes.findIndex( + (hash, index) => chain[firstHeight + index] !== hash + ); + const sharedCount = + firstDivergingIndex === -1 ? newHashes.length : firstDivergingIndex; + const divergeHeight = firstHeight + sharedCount; const staleHashes = chain.splice( - firstHeight, - newHashes.length, - ...newHashes + divergeHeight, + newHashes.length - sharedCount, + ...newHashes.slice(sharedCount) ); if (staleHashes.length !== 0) { this.logger.warn( `${nodeName}: removed ${staleHashes.length} stale block(s) from the block tree.` ); - this.onStaleBlocks(staleHashes, firstHeight, nodeName); + this.onStaleBlocks(staleHashes, divergeHeight, nodeName); } this.logger.debug( `${nodeName}: added ${ From efdfad85bba7a9dd75e135c7fd05e03a76814777 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 21 Oct 2026 09:19:53 +0100 Subject: [PATCH 59/63] fix(agent): resync node mempools after reorgs and on connect (fixes #81) (#8) After a reorg and on (re)connect, the agent requests each node's mempool (BIP35, rate-limited) so transactions returned to the mempool are recorded again. An expiry now carries the validated_at it was scheduled for, so a validation restored by the resync is not archived by a stale expiry timer. Adds e2e and database regression tests. Fixes bitauth/chaingraph#81. Co-Authored-By: Claude Opus 5.5 --- defaults.env | 13 ++ src/agent.ts | 36 ++++ src/components/mempool-resync.spec.ts | 276 ++++++++++++++++++++++++++ src/components/mempool-resync.ts | 206 +++++++++++++++++++ src/config.ts | 32 +++ src/db.ts | 16 +- src/e2e/e2e.spec.ts | 156 ++++++++++++++- src/e2e/mempool-cleanup.spec.ts | 46 +++++ src/types/bitcore-p2p-cash.d.ts | 3 + 9 files changed, 778 insertions(+), 6 deletions(-) create mode 100644 src/components/mempool-resync.spec.ts create mode 100644 src/components/mempool-resync.ts diff --git a/defaults.env b/defaults.env index efb79e71..35f826ee 100644 --- a/defaults.env +++ b/defaults.env @@ -31,6 +31,19 @@ CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE=10000 CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS=1209600000 CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS=3600000 +# BCHN does not re-announce transactions that a re-organization returns to its +# mempool. When enabled, the agent sends each trusted node a BIP35 `mempool` +# request after mempool tracking is enabled, after the node reconnects, and +# after each re-organization reported by that node, so the answering `inv` +# re-records `node_transaction` rows. Replies only add rows. Requests to each +# node are rate-limited to one per CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS; +# requests during that window are coalesced into one request when it ends. +# The node must answer `mempool` requests: BCHN does when NODE_BLOOM is enabled +# (`-peerbloomfilters`, the default) or the agent has the `mempool` permission +# (e.g. `-whitelist=mempool@`). See bitauth/chaingraph#81. +CHAINGRAPH_MEMPOOL_RESYNC=true +CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS=30000 + # A mapping of network magic bytes to hex-encoded genesis blocks. # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated. # E.g. CHAINGRAPH_GENESIS_BLOCKS=e3e1f3e8:rawblockhex,deadbeef:rawblockhex diff --git a/src/agent.ts b/src/agent.ts index 17a50fbf..d5e6cbdc 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -20,6 +20,7 @@ import { import { BlockBuffer } from './components/block-buffer.js'; import { BlockTree } from './components/block-tree.js'; import type { indexDefinitions } from './components/db-utils.js'; +import { MempoolResync } from './components/mempool-resync.js'; import { SyncState } from './components/sync-state.js'; import { formatBytes, @@ -32,6 +33,8 @@ import { chaingraphUserAgent, genesisBlocks, incompleteBlockRepairBatchSize, + mempoolResync, + mempoolResyncMinIntervalMs, mempoolTransactionExpirationMs, mempoolTransactionExpirationScanIntervalMs, postgresMaxConnections, @@ -322,6 +325,32 @@ export class Agent { nodesByInternalId = [] as [nodeName: string, node: Node][]; + /** + * Requests the mempool of trusted nodes (BIP35) after mempool tracking is + * enabled, on reconnection, and after re-organizations, so transactions + * which nodes don't re-announce (e.g. those returned to the mempool by a + * re-organization) are recorded in `node_transaction`. See + * https://github.com/bitauth/chaingraph/issues/81. + */ + mempoolResync = new MempoolResync({ + enabled: mempoolResync, + getReadyNode: (nodeName) => { + const node = this.nodes[nodeName]; + return node?.internalId !== undefined && node.peer.status === 'ready' + ? node + : undefined; + }, + minimumIntervalMs: mempoolResyncMinIntervalMs, + onRequestSent: (nodeName, triggers) => { + this.logger.info( + `${nodeName}: requested node mempool (BIP35) – triggers: ${triggers.join( + ', ' + )}` + ); + }, + transactionCache: this.transactionCache, + }); + heartbeatInterval: NodeJS.Timeout; scheduledBlockBufferFill = false; @@ -612,6 +641,7 @@ export class Agent { } nodeRegisteredResolver(); this.requestHeaders(node.name); + this.mempoolResync.handleConnect(node.name); }); }); @@ -942,6 +972,9 @@ export class Agent { } this.logger.info('Agent: enabled mempool tracking.'); this.saveInboundTransactions = true; + this.mempoolResync.enableMempoolTracking( + Object.keys(this.nodes) + ); this.scheduleIncompleteBlockRepair(); this.scheduleMempoolTransactionExpirationScan(); }); @@ -1118,6 +1151,7 @@ export class Agent { nodeInternalId: transaction.nodeInternalId, replacedAt: transaction.expiresAt, transactionInternalId: transaction.transactionInternalId, + validatedAt: transaction.validatedAt, }); if (!archived) { this.logger.debug( @@ -1918,6 +1952,7 @@ export class Agent { staleChain, `${nodeName}: re-organization detected beginning at height: ${firstHeight}. The following stale blocks were removed:` ); + this.mempoolResync.handleReorganization(nodeName); this.scheduleBlockBufferFill(); }) .catch((err) => { @@ -2242,6 +2277,7 @@ export class Agent { return this.shutdownPromise; } this.willShutdown = true; + this.mempoolResync.stop(); clearInterval(eventLoopDurationInterval); clearInterval(this.heartbeatInterval); if (this.incompleteBlockRepairTimeout !== undefined) { diff --git a/src/components/mempool-resync.spec.ts b/src/components/mempool-resync.spec.ts new file mode 100644 index 00000000..f5e51d01 --- /dev/null +++ b/src/components/mempool-resync.spec.ts @@ -0,0 +1,276 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import bitcoreP2pCash from '@chaingraph/bitcore-p2p-cash'; +import type { Message } from '@chaingraph/bitcore-p2p-cash'; +import test from 'ava'; +import LRU from 'lru-cache'; + +import type { + MempoolResyncCacheItem, + MempoolResyncOptions, + MempoolResyncTrigger, +} from './mempool-resync.js'; +import { + forgetSavedAcknowledgementsFromNode, + MempoolResync, +} from './mempool-resync.js'; + +interface MockNode { + name: string; + peer: InstanceType; + ready: boolean; + sentMessages: Message[]; +} + +const createMockNode = (name: string): MockNode => { + const sentMessages: Message[] = []; + const peer = new bitcoreP2pCash.Peer({}); + peer.sendMessage = (message) => { + sentMessages.push(message); + }; + return { name, peer, ready: true, sentMessages }; +}; + +const commandStart = 4; +const commandEnd = 16; +const commandsSent = (node: MockNode) => + node.sentMessages.map((message) => + message + .toBuffer() + .subarray(commandStart, commandEnd) + .toString('latin1') + .replace(/\0+$/u, '') + ); + +const setup = (options: Partial> = {}) => { + const nodes = { + node1: createMockNode('node1'), + node2: createMockNode('node2'), + }; + const transactionCache = new LRU>({ + max: 10, + }); + const clock = { now: 0 }; + const timers: { callback: () => void; dueAt: number; cleared: boolean }[] = + []; + const advance = (ms: number) => { + clock.now += ms; + timers + .filter((timer) => !timer.cleared && timer.dueAt <= clock.now) + .forEach((timer) => { + timer.cleared = true; + timer.callback(); + }); + }; + const requestsSent: [string, MempoolResyncTrigger[]][] = []; + const mempoolResync = new MempoolResync({ + clearTimer: (timer) => { + (timer as (typeof timers)[number]).cleared = true; + }, + enabled: true, + getReadyNode: (nodeName) => { + const node = nodes[nodeName as keyof typeof nodes]; + return node.ready ? node : undefined; + }, + minimumIntervalMs: 1000, + now: () => clock.now, + onRequestSent: (nodeName, triggers) => { + requestsSent.push([nodeName, triggers]); + }, + setTimer: (callback, ms) => { + const timer = { callback, cleared: false, dueAt: clock.now + ms }; + timers.push(timer); + return timer; + }, + transactionCache, + ...options, + }); + return { + advance, + mempoolResync, + nodes, + requestsSent, + timers, + transactionCache, + }; +}; + +test('sends the BIP35 mempool wire message', (t) => { + const { mempoolResync, nodes } = setup(); + mempoolResync.enableMempoolTracking(['node1']); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + const wire = nodes.node1.sentMessages[0]!.toBuffer(); + t.is(wire.subarray(4, 16).toString('hex'), '6d656d706f6f6c0000000000'); + t.is(wire.readUInt32LE(16), 0); + t.is(wire.length, 24); +}); + +test('requests every node once mempool tracking is enabled', (t) => { + const { mempoolResync, nodes, requestsSent } = setup(); + mempoolResync.enableMempoolTracking(['node1', 'node2']); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + t.deepEqual(commandsSent(nodes.node2), ['mempool']); + t.deepEqual(requestsSent, [ + ['node1', ['mempool-tracking-enabled']], + ['node2', ['mempool-tracking-enabled']], + ]); +}); + +test('requests a node on (re)connect, but only after mempool tracking is enabled', (t) => { + const { advance, mempoolResync, nodes, requestsSent } = setup(); + mempoolResync.handleConnect('node1'); + t.deepEqual(commandsSent(nodes.node1), []); + mempoolResync.enableMempoolTracking([]); + mempoolResync.handleConnect('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + advance(1000); + mempoolResync.handleConnect('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool']); + t.deepEqual(requestsSent, [ + ['node1', ['connect']], + ['node1', ['connect']], + ]); +}); + +test('requests a node after a re-organization, but only after mempool tracking is enabled', (t) => { + const { mempoolResync, nodes, requestsSent } = setup(); + mempoolResync.handleReorganization('node2'); + t.deepEqual(commandsSent(nodes.node2), []); + mempoolResync.enableMempoolTracking([]); + mempoolResync.handleReorganization('node2'); + t.deepEqual(commandsSent(nodes.node1), []); + t.deepEqual(commandsSent(nodes.node2), ['mempool']); + t.deepEqual(requestsSent, [['node2', ['reorganization']]]); +}); + +test('sends nothing when disabled', (t) => { + const { mempoolResync, nodes, timers } = setup({ enabled: false }); + mempoolResync.enableMempoolTracking(['node1', 'node2']); + mempoolResync.handleConnect('node1'); + mempoolResync.handleReorganization('node2'); + t.deepEqual(commandsSent(nodes.node1), []); + t.deepEqual(commandsSent(nodes.node2), []); + t.is(timers.length, 0); +}); + +test('forgets saved acknowledgements from the node before each request', (t) => { + const { mempoolResync, nodes, transactionCache } = setup(); + const { node1, node2 } = nodes; + transactionCache.set('saved-both', { db: true, nodes: [node1, node2] }); + transactionCache.set('saved-node1', { db: true, nodes: [node1] }); + transactionCache.set('saved-node2', { db: true, nodes: [node2] }); + transactionCache.set('unsaved', { db: false, nodes: [node1, node2] }); + node1.peer.sendMessage = () => { + t.deepEqual(transactionCache.get('saved-node1')!.nodes, []); + }; + mempoolResync.handleConnect('node1'); + mempoolResync.enableMempoolTracking(['node1']); + t.deepEqual(transactionCache.get('saved-both'), { + db: true, + nodes: [node2], + }); + t.deepEqual(transactionCache.get('saved-node1'), { db: true, nodes: [] }); + t.deepEqual(transactionCache.get('saved-node2'), { + db: true, + nodes: [node2], + }); + t.deepEqual(transactionCache.get('unsaved'), { + db: false, + nodes: [node1, node2], + }); + t.is(transactionCache.size, 4); + t.plan(6); +}); + +test('forgetSavedAcknowledgementsFromNode never removes cache entries', (t) => { + const { nodes, transactionCache } = setup(); + transactionCache.set('a', { db: true, nodes: [nodes.node1] }); + forgetSavedAcknowledgementsFromNode(nodes.node1, transactionCache); + forgetSavedAcknowledgementsFromNode(nodes.node1, transactionCache); + t.deepEqual(transactionCache.get('a'), { db: true, nodes: [] }); + t.is(transactionCache.size, 1); +}); + +test('rate-limits each node, coalescing triggers into one deferred request', (t) => { + const { advance, mempoolResync, nodes, requestsSent, timers } = setup(); + mempoolResync.enableMempoolTracking(['node1', 'node2']); + advance(100); + mempoolResync.handleReorganization('node1'); + mempoolResync.handleConnect('node1'); + mempoolResync.handleReorganization('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + t.is(timers.length, 1); + advance(899); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + advance(1); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool']); + t.deepEqual(commandsSent(nodes.node2), ['mempool']); + t.deepEqual(requestsSent.slice(2), [ + ['node1', ['reorganization', 'connect']], + ]); + advance(500); + mempoolResync.handleReorganization('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool']); + advance(500); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool', 'mempool']); +}); + +test('a flapping node is requested at most once per interval', (t) => { + const { advance, mempoolResync, nodes } = setup(); + mempoolResync.enableMempoolTracking([]); + Array.from({ length: 100 }).forEach(() => { + mempoolResync.handleConnect('node1'); + advance(50); + }); + /** + * 100 reconnects over 5 seconds: sent at 0, 1000, 2000, 3000, 4000, and + * (coalescing the reconnects after 4000) 5000. + */ + t.is(commandsSent(nodes.node1).length, 6); +}); + +test('skips a deferred request if the node disconnected', (t) => { + const { advance, mempoolResync, nodes, requestsSent } = setup(); + mempoolResync.enableMempoolTracking(['node1']); + mempoolResync.handleReorganization('node1'); + nodes.node1.ready = false; + advance(1000); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + nodes.node1.ready = true; + mempoolResync.handleConnect('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool']); + t.deepEqual(requestsSent, [ + ['node1', ['mempool-tracking-enabled']], + ['node1', ['connect']], + ]); +}); + +test('stop cancels deferred requests and ignores later triggers', (t) => { + const { advance, mempoolResync, nodes, timers } = setup(); + mempoolResync.enableMempoolTracking(['node1']); + mempoolResync.handleReorganization('node1'); + mempoolResync.stop(); + t.true(timers[0]!.cleared); + advance(5000); + mempoolResync.handleReorganization('node1'); + mempoolResync.handleConnect('node2'); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + t.deepEqual(commandsSent(nodes.node2), []); +}); + +test('uses real timers by default', async (t) => { + const node = createMockNode('node1'); + const mempoolResync = new MempoolResync({ + enabled: true, + getReadyNode: () => node, + minimumIntervalMs: 20, + transactionCache: [], + }); + mempoolResync.enableMempoolTracking(['node1']); + mempoolResync.handleReorganization('node1'); + t.deepEqual(commandsSent(node), ['mempool']); + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + t.deepEqual(commandsSent(node), ['mempool', 'mempool']); + mempoolResync.stop(); +}); diff --git a/src/components/mempool-resync.ts b/src/components/mempool-resync.ts new file mode 100644 index 00000000..6a3bbe46 --- /dev/null +++ b/src/components/mempool-resync.ts @@ -0,0 +1,206 @@ +import type { Peer } from '@chaingraph/bitcore-p2p-cash'; + +/** + * The events after which a node's mempool is requested. + */ +export type MempoolResyncTrigger = + | 'connect' + | 'mempool-tracking-enabled' + | 'reorganization'; + +export interface MempoolResyncNode { + peer: Pick; +} + +export interface MempoolResyncCacheItem { + db: boolean; + nodes: Node[]; +} + +export interface MempoolResyncOptions { + /** + * If `false`, no `mempool` requests are ever sent. + */ + enabled: boolean; + /** + * The minimum number of milliseconds between two `mempool` requests to the + * same node. Requests made during this window are coalesced into a single + * request sent when the window ends. + */ + minimumIntervalMs: number; + /** + * Return the node if it is connected and registered (ready to receive a + * `mempool` request and to have its announcements recorded), otherwise + * `undefined`. + */ + getReadyNode: (nodeName: string) => Node | undefined; + /** + * The agent's transaction cache. Before each request, the node is removed + * from the acknowledgements of already-saved transactions. + */ + transactionCache: Iterable<[string, MempoolResyncCacheItem]>; + /** + * Called after each `mempool` request is sent. + */ + onRequestSent?: (nodeName: string, triggers: MempoolResyncTrigger[]) => void; + now?: () => number; + setTimer?: (callback: () => void, ms: number) => unknown; + clearTimer?: (timer: unknown) => void; +} + +/** + * Remove `node` from the cached acknowledgements of every transaction which + * has already been saved to the database, so a later announcement of the + * transaction by this node is recorded again (rather than ignored as a + * duplicate). Acknowledgements of transactions not yet saved are kept: they + * are still needed to record validations when the transaction is saved. + */ +export const forgetSavedAcknowledgementsFromNode = ( + node: Node, + transactionCache: Iterable<[string, MempoolResyncCacheItem]> +) => { + // eslint-disable-next-line functional/no-loop-statement + for (const [, cacheItem] of transactionCache) { + if (cacheItem.db && cacheItem.nodes.includes(node)) { + cacheItem.nodes = cacheItem.nodes.filter((source) => source !== node); + } + } +}; + +/** + * Requests the mempool of trusted nodes with BIP35 `mempool` messages. + * + * Nodes don't announce transactions which a re-organization returns to their + * mempool, and the agent otherwise only hears about mempool transactions from + * unsolicited announcements. To keep `node_transaction` current, the node's + * mempool is requested: + * - for every connected node once mempool tracking is enabled, + * - when a node (re)connects after mempool tracking is enabled, and + * - after a re-organization reported by the node has been handled. + * + * The node answers with `inv` messages, which are handled by the existing + * announcement path. BIP35 replies may be split across many `inv` messages + * and have no completion marker, so a reply can only ever add + * `node_transaction` rows – a missing hash never proves absence. + * + * Requests are rate-limited per node: at most one request is sent per + * `minimumIntervalMs`, and requests during that window are coalesced into one + * request sent at the end of the window (so a re-organization shortly after a + * request is never lost, and a flapping node can't cause a request storm). + */ +export class MempoolResync { + private readonly options: Required>; + + private trackingEnabled = false; + + private stopped = false; + + private readonly lastRequestAt = new Map(); + + private readonly deferredRequests = new Map< + string, + { timer: unknown; triggers: Set } + >(); + + constructor(options: MempoolResyncOptions) { + this.options = { + clearTimer: (timer) => { + clearTimeout(timer as ReturnType); + }, + now: Date.now, + onRequestSent: () => undefined, + setTimer: (callback, ms) => setTimeout(callback, ms), + ...options, + }; + } + + /** + * Call when mempool tracking is enabled: requests the mempool of each given + * node, and enables requests on later connections and re-organizations. + */ + enableMempoolTracking(nodeNames: string[]) { + this.trackingEnabled = true; + nodeNames.forEach((nodeName) => { + this.request(nodeName, 'mempool-tracking-enabled'); + }); + } + + /** + * Call after a node has (re)connected and been registered. + */ + handleConnect(nodeName: string) { + this.request(nodeName, 'connect'); + } + + /** + * Call after a re-organization reported by the node has been handled. + */ + handleReorganization(nodeName: string) { + this.request(nodeName, 'reorganization'); + } + + /** + * Cancel any deferred requests and ignore all future triggers. + */ + stop() { + this.stopped = true; + this.deferredRequests.forEach(({ timer }) => { + this.options.clearTimer(timer); + }); + this.deferredRequests.clear(); + } + + private isActive() { + return this.options.enabled && this.trackingEnabled && !this.stopped; + } + + private millisecondsUntilNextRequestAllowed(nodeName: string) { + const lastRequestAt = this.lastRequestAt.get(nodeName); + if (lastRequestAt === undefined) { + return 0; + } + return Math.max( + 0, + lastRequestAt + this.options.minimumIntervalMs - this.options.now() + ); + } + + private request(nodeName: string, trigger: MempoolResyncTrigger) { + if (!this.isActive()) { + return; + } + const deferred = this.deferredRequests.get(nodeName); + if (deferred !== undefined) { + deferred.triggers.add(trigger); + return; + } + const delay = this.millisecondsUntilNextRequestAllowed(nodeName); + if (delay === 0) { + this.send(nodeName, [trigger]); + return; + } + const triggers = new Set([trigger]); + const timer = this.options.setTimer(() => { + this.deferredRequests.delete(nodeName); + this.send(nodeName, [...triggers]); + }, delay); + this.deferredRequests.set(nodeName, { timer, triggers }); + } + + private send(nodeName: string, triggers: MempoolResyncTrigger[]) { + if (this.stopped) { + return; + } + const node = this.options.getReadyNode(nodeName); + if (node === undefined) { + /** + * The node is disconnected: it will be requested again on reconnect. + */ + return; + } + forgetSavedAcknowledgementsFromNode(node, this.options.transactionCache); + node.peer.sendMessage(new node.peer.messages.MemPool()); + this.lastRequestAt.set(nodeName, this.options.now()); + this.options.onRequestSent(nodeName, triggers); + } +} diff --git a/src/config.ts b/src/config.ts index 00c37a5a..ff2446b2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -42,6 +42,8 @@ const expectedOptions = [ 'CHAINGRAPH_LOG_LEVEL_STDOUT', 'CHAINGRAPH_LOG_LEVEL_PATH', 'CHAINGRAPH_LOG_PATH', + 'CHAINGRAPH_MEMPOOL_RESYNC', + 'CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS', 'CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS', 'CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS', 'CHAINGRAPH_POSTGRES_CONNECTION_STRING', @@ -154,6 +156,34 @@ if ( ); } +if ( + configuration.CHAINGRAPH_MEMPOOL_RESYNC.toLowerCase() !== 'true' && + configuration.CHAINGRAPH_MEMPOOL_RESYNC.toLowerCase() !== 'false' +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Invalid value provided in the 'CHAINGRAPH_MEMPOOL_RESYNC' environment variable: ${configuration.CHAINGRAPH_MEMPOOL_RESYNC}. Must be one of the following: true, false` + ); +} +/** + * Set via the `CHAINGRAPH_MEMPOOL_RESYNC` environment variable. + */ +const mempoolResync = + configuration.CHAINGRAPH_MEMPOOL_RESYNC.toLowerCase() === 'true'; + +const mempoolResyncMinIntervalMs = Number( + configuration.CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS +); +if ( + !Number.isInteger(mempoolResyncMinIntervalMs) || + mempoolResyncMinIntervalMs < 0 +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + 'The CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS environment variable must be an integer greater than or equal to 0.' + ); +} + const extendTildeAndResolvePath = (path: string) => path.startsWith('~') ? resolve(join(homedir(), path.slice(1))) @@ -419,6 +449,8 @@ export { chaingraphUserAgent, genesisBlocks, incompleteBlockRepairBatchSize, + mempoolResync, + mempoolResyncMinIntervalMs, mempoolTransactionExpirationMs, mempoolTransactionExpirationScanIntervalMs, postgresMaxConnections, diff --git a/src/db.ts b/src/db.ts index 0c396bff..b6bbb8ad 100644 --- a/src/db.ts +++ b/src/db.ts @@ -429,6 +429,12 @@ export interface MempoolTransactionExpiry { nodeInternalId: number; replacedAt: Date; transactionInternalId: number; + /** + * The `validated_at` of the validation this expiry was scheduled for. When + * set, a row validated again later (e.g. restored by a mempool resync after a + * re-org) is left alone; it expires at its own deadline. + */ + validatedAt?: Date; } // cspell:words xact unnest @@ -584,9 +590,9 @@ SELECT 1 }>( /* sql */ ` WITH RECURSIVE expiry AS ( - SELECT node_internal_id, transaction_internal_id, replaced_at - FROM unnest($1::integer[], $2::bigint[], $3::timestamp[]) - AS expiry (node_internal_id, transaction_internal_id, replaced_at) + SELECT node_internal_id, transaction_internal_id, replaced_at, validated_at + FROM unnest($1::integer[], $2::bigint[], $3::timestamp[], $4::timestamp[]) + AS expiry (node_internal_id, transaction_internal_id, replaced_at, validated_at) ), expired AS ( SELECT expiry.node_internal_id, @@ -603,6 +609,9 @@ expired AS ( ) resolution WHERE NOT resolution.accepted AND NOT resolution.replaced + -- validated_at has microseconds; the scheduled Date has milliseconds + AND (expiry.validated_at IS NULL + OR date_trunc('milliseconds', node_transaction.validated_at) <= expiry.validated_at) ), invalidated (node_internal_id, transaction_internal_id, replaced_at) AS ( SELECT node_internal_id, transaction_internal_id, replaced_at @@ -693,6 +702,7 @@ SELECT complete.complete, nodeInternalIds, transactionInternalIds, sorted.map((expiry) => expiry.replacedAt.toISOString()), + sorted.map((expiry) => expiry.validatedAt?.toISOString() ?? null), ] ); if (result.rows[0]?.complete !== true) { diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index d6a28207..0acef632 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -166,6 +166,7 @@ const e2eEnvVariables = { CHAINGRAPH_INTERNAL_API_PORT: chaingraphInternalApiPort, CHAINGRAPH_LOG_FIREHOSE: logP2pMessage.toString(), CHAINGRAPH_LOG_PATH: chaingraphE2eLogPath, + CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS: '100', CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS: '100', CHAINGRAPH_POSTGRES_CONNECTION_STRING: postgresE2eConnectionStringTestDb, CHAINGRAPH_TRUSTED_NODES: e2eTrustedNodesSet1, @@ -191,6 +192,40 @@ const peers = { node3: placeholder, }; +/** + * The transaction hashes (hex, as displayed) each mock node announces in reply + * to a BIP35 `mempool` request, and the number of requests each has received. + */ +const mempoolReplies: { [nodeName in keyof typeof peers]: string[] } = { + node1: [], + node2: [], + node3: [], +}; +const mempoolRequestCounts: { [nodeName in keyof typeof peers]: number } = { + node1: 0, + node2: 0, + node3: 0, +}; +const respondToMempoolRequests = (nodeName: keyof typeof peers, peer: Peer) => { + peer.on('mempool', () => { + mempoolRequestCounts[nodeName] += 1; + logger.debug( + `e2e: ${nodeName} received mempool request, announcing: ${mempoolReplies[ + nodeName + ].join(', ')}` + ); + /** + * BIP35 replies may be split across multiple `inv` messages; announce each + * transaction in its own message. + */ + mempoolReplies[nodeName].forEach((hash) => { + peer.sendMessage( + peer.messages.Inventory.forTransaction(Buffer.from(hash, 'hex')) + ); + }); + }); +}; + test.beforeEach((t) => { logger.debug(`Starting test: ${t.title}`); }); @@ -243,15 +278,24 @@ test.before(async () => { }; node1.on('peerready', (peer) => { logPeerConnection(`node1`, peer); - if (!peer.subversion.includes('tx-broadcast')) peers.node1 = peer; + if (!peer.subversion.includes('tx-broadcast')) { + peers.node1 = peer; + respondToMempoolRequests('node1', peer); + } }); node2.on('peerready', (peer) => { logPeerConnection(`node2`, peer); - if (!peer.subversion.includes('tx-broadcast')) peers.node2 = peer; + if (!peer.subversion.includes('tx-broadcast')) { + peers.node2 = peer; + respondToMempoolRequests('node2', peer); + } }); node3.on('peerready', (peer) => { logPeerConnection(`node3`, peer); - if (!peer.subversion.includes('tx-broadcast')) peers.node3 = peer; + if (!peer.subversion.includes('tx-broadcast')) { + peers.node3 = peer; + respondToMempoolRequests('node3', peer); + } }); if (logP2pMessage) { @@ -837,6 +881,27 @@ test.serial('[e2e] completes initial sync', async (t) => { t.pass(); }); +test.serial( + '[e2e] requests the mempool of each node once mempool tracking is enabled', + async (t) => { + await waitForStdout('Agent: enabled mempool tracking.'); + await waitForStdout( + 'node1: requested node mempool (BIP35) – triggers: mempool-tracking-enabled' + ); + await waitForStdout( + 'node2: requested node mempool (BIP35) – triggers: mempool-tracking-enabled' + ); + await waitForStdout( + 'node3: requested node mempool (BIP35) – triggers: mempool-tracking-enabled' + ); + const delay = 500; + await sleep(delay); + t.true(mempoolRequestCounts.node1 >= 1); + t.true(mempoolRequestCounts.node2 >= 1); + t.true(mempoolRequestCounts.node3 >= 1); + } +); + test.serial('[e2e] creates expected indexes after initial sync', async (t) => { await waitForStdout('Agent: all managed indexes have been created.'); await waitForStdout('Agent: enabled mempool tracking.'); @@ -2081,6 +2146,73 @@ test.serial( } ); +const getNodeValidationCount = async (nodeName: string, hash: string) => + Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*) FROM node_transaction + INNER JOIN node + ON node.internal_id = node_transaction.node_internal_id + INNER JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + WHERE node.name = $1 AND transaction.hash = $2; + `, + [nodeName, hexToBin(hash)] + ) + ).rows[0]!.count + ); +const deleteNodeValidation = async (nodeName: string, hash: string) => + client.query( + /* sql */ ` + DELETE FROM node_transaction + USING node, transaction + WHERE node_transaction.node_internal_id = node.internal_id + AND node_transaction.transaction_internal_id = transaction.internal_id + AND node.name = $1 + AND transaction.hash = $2; + `, + [nodeName, hexToBin(hash)] + ); +const nodeValidationPollingAttempts = 50; +const nodeValidationPollingIntervalMs = 100; +/** + * Wait for the node's `node_transaction` row for the transaction, returning + * the final count of matching rows (`0` if it never appeared). + */ +const waitForNodeValidation = async ( + nodeName: string, + hash: string, + remainingAttempts = nodeValidationPollingAttempts +): Promise => { + const count = await getNodeValidationCount(nodeName, hash); + if (count !== 0 || remainingAttempts === 0) { + return count; + } + await sleep(nodeValidationPollingIntervalMs); + return waitForNodeValidation(nodeName, hash, remainingAttempts - 1); +}; + +/** + * Model https://github.com/bitauth/chaingraph/issues/81: node3 accepted a + * transaction (its `node_transaction` row was saved and the agent cached + * node3's acknowledgement), the transaction was later confirmed (archiving the + * row), then a re-organization returned it to node3's mempool. Nodes don't + * re-announce such transactions; they only appear in replies to `mempool`. + */ +test.serial( + '[e2e] prepares a transaction for return to the mempool by a re-org', + async (t) => { + peers.node3.sendMessage( + new peers.node3.messages.Transaction(new Transaction(halTxRaw)) + ); + t.is(await waitForNodeValidation('node3', halTxHash), 1); + await deleteNodeValidation('node3', halTxHash); + t.is(await getNodeValidationCount('node3', halTxHash), 0); + mempoolReplies.node3 = [halTxHash]; + } +); + test.serial('[e2e] handles re-org of a single block', async (t) => { newBlocks('node1', [tipA[1]!]); newBlocks('node2', [tipB[1]!]); @@ -2102,6 +2234,24 @@ test.serial('[e2e] handles re-org of a single block', async (t) => { ); }); +test.serial( + '[e2e] re-records transactions returned to the mempool by a re-org', + async (t) => { + await waitForStdout( + 'node3: requested node mempool (BIP35) – triggers: reorganization' + ); + t.is(await waitForNodeValidation('node3', halTxHash), 1); + /** + * Clean up: stop announcing the transaction, wait for any pending replies, + * then remove node3's validation so later tests are unaffected. + */ + mempoolReplies.node3 = []; + const delay = 500; + await sleep(delay); + await deleteNodeValidation('node3', halTxHash); + } +); + test.serial('[e2e] new block saved after reorg', async (t) => { const acceptedBlocks = ( await client.query<{ diff --git a/src/e2e/mempool-cleanup.spec.ts b/src/e2e/mempool-cleanup.spec.ts index 839c7c89..7a13ec3b 100644 --- a/src/e2e/mempool-cleanup.spec.ts +++ b/src/e2e/mempool-cleanup.spec.ts @@ -469,3 +469,49 @@ INSERT INTO node_transaction (node_internal_id, transaction_internal_id) t.deepEqual((await pool.query(membershipQuery)).rows, [{ node: 1, tx: 3 }]); } ); + +/* + * Regression: an expiry scheduled for one validation also archived a later + * validation of the same transaction (e.g. one restored by a mempool resync + * after the first was confirmed and a re-org removed the block), and through the + * history trigger that validation's descendants. + */ +test.serial( + '[e2e] expiry leaves a transaction validated again after it was scheduled', + async (t) => { + const { pool, archiveMempoolTransactions } = t.context.db; + // 2 spends 1; node 1 re-validated 1 after the expiry was scheduled + await pool.query(/* sql */ ` +INSERT INTO input SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1; +INSERT INTO node_transaction VALUES + (1, 1, '2026-01-02 00:00:00.123456'), (1, 2, '2026-01-02'), + (2, 1, '2026-01-01 00:00:00.123456'); +`); + const replacedAt = new Date('2026-01-01T01:00:00.000Z'); + const archived = await archiveMempoolTransactions([ + { + nodeInternalId: 1, + replacedAt, + transactionInternalId: 1, + validatedAt: new Date('2026-01-01T00:00:00.123Z'), + }, + { + nodeInternalId: 2, + replacedAt, + transactionInternalId: 1, + validatedAt: new Date('2026-01-01T00:00:00.123Z'), + }, + ]); + t.deepEqual( + archived.map((expiry) => ({ node: expiry.nodeInternalId })), + [{ node: 2 }] + ); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 1, tx: 1 }, + { node: 1, tx: 2 }, + ]); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 2, replaced: '2026-01-01 01:00:00', tx: 1 }, + ]); + } +); diff --git a/src/types/bitcore-p2p-cash.d.ts b/src/types/bitcore-p2p-cash.d.ts index d330ace2..e339c271 100644 --- a/src/types/bitcore-p2p-cash.d.ts +++ b/src/types/bitcore-p2p-cash.d.ts @@ -70,6 +70,7 @@ declare module '@chaingraph/bitcore-p2p-cash' { inventory: BitcoreInventory[]; } export class SendHeadersMessage extends Message {} + export class MemPoolMessage extends Message {} export class TransactionMessage extends Message { transaction: Transaction; @@ -88,6 +89,7 @@ declare module '@chaingraph/bitcore-p2p-cash' { getdata: () => void; headers: (message: HeadersMessage) => void; inv: (message: InventoryMessage) => void; + mempool: (message: MemPoolMessage) => void; ready: () => void; tx: (message: TransactionMessage) => void; } @@ -114,6 +116,7 @@ declare module '@chaingraph/bitcore-p2p-cash' { forTransaction: (transactionHash: Buffer | string) => InventoryMessage; }; SendHeaders: typeof SendHeadersMessage; + MemPool: typeof MemPoolMessage; Transaction: typeof TransactionMessage; }; From e3acbd478e07410133d2d700401930b44c3d3641 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 21 Oct 2026 09:23:31 +0100 Subject: [PATCH 60/63] perf(agent): stop re-serialising blocks/transactions during conversion (#10) Block conversion reuses the bytes received from the node instead of re-serialising blocks and transactions: transaction hashes and sizes come from the wire bytes, one-shot hashing is used where available and transaction fields are read directly. Adds parity tests for decoding and conversion. Co-Authored-By: Claude Opus 5.5 --- .cspell.json | 1 + src/agent.ts | 6 +- src/bitcore.spec.ts | 197 ++++++++++++++++++++++++++++++ src/bitcore.ts | 204 +++++++++++++++++++++++++++----- src/types/bitcore-lib-cash.d.ts | 40 ++++++- src/types/bitcore-p2p-cash.d.ts | 7 ++ 6 files changed, 422 insertions(+), 33 deletions(-) create mode 100644 src/bitcore.spec.ts diff --git a/.cspell.json b/.cspell.json index 779f066e..6a7c1519 100644 --- a/.cspell.json +++ b/.cspell.json @@ -105,6 +105,7 @@ "upserting", "UTXO", "UTXOs", + "varint", "verack", "xprivkey", "xpubkey" diff --git a/src/agent.ts b/src/agent.ts index d5e6cbdc..b2df6738 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -16,6 +16,7 @@ import type pino from 'pino'; import { bitcoreBlockToChaingraphBlock, bitcoreTransactionToChaingraphTransaction, + createPeerMessages, } from './bitcore.js'; import { BlockBuffer } from './components/block-buffer.js'; import { BlockTree } from './components/block-tree.js'; @@ -534,7 +535,10 @@ export class Agent { subversion: chaingraphUserAgent, version: chaingraphProtocolVersion, }; - const peer = new Peer(peerConfig); + const peer = new Peer({ + ...peerConfig, + messages: createPeerMessages(node.networkMagicHex), + }); const peerTxBroadcastConnection = new Peer({ ...peerConfig, subversion: `${chaingraphUserAgent}tx-broadcast/`, diff --git a/src/bitcore.spec.ts b/src/bitcore.spec.ts new file mode 100644 index 00000000..96bafd8c --- /dev/null +++ b/src/bitcore.spec.ts @@ -0,0 +1,197 @@ +import type { + BitcoreBlock, + BitcoreTransaction, +} from '@chaingraph/bitcore-p2p-cash'; +import test from 'ava'; + +import { + bitcoreBlockToChaingraphBlock, + bitcoreTransactionToChaingraphTransaction, + decodeBlockRetainingWireBytes, + messages, +} from './bitcore.js'; +import { + Block, + chipnetCashTokensTx, + chipnetCashTokensTxHash, + generateMockchain, + genesisBlockRaw, + halTxHash, + halTxRaw, + halTxSpent, + halTxSpentRaw, + testnetGenesisBlockRaw, + Transaction, +} from './e2e/e2e.spec.mockchain.helper.js'; +import type { + ChaingraphBlock, + ChaingraphTransaction, +} from './types/chaingraph.js'; + +/** + * The implementation prior to retaining wire bytes (re-serializes every + * transaction and the full block), kept as a reference for parity tests. + */ +const referenceTransactionConversion = ( + bitcoreTransaction: BitcoreTransaction +): ChaingraphTransaction => { + const txObject = bitcoreTransaction.toObject(); + const isCoinbase = bitcoreTransaction.isCoinbase(); + return { + hash: txObject.hash, + inputs: txObject.inputs.map((input) => ({ + outpointIndex: input.outputIndex, + outpointTransactionHash: input.prevTxId, + sequenceNumber: input.sequenceNumber, + unlockingBytecode: input.script, + })), + isCoinbase, + locktime: txObject.nLockTime, + outputs: txObject.outputs.map((output) => ({ + fungibleTokenAmount: + output.tokenData === undefined + ? undefined + : BigInt(output.tokenData.amount), + lockingBytecode: output.script, + nonfungibleTokenCapability: output.tokenData?.nft?.capability, + nonfungibleTokenCommitment: output.tokenData?.nft?.commitment, + tokenCategory: output.tokenData?.category, + valueSatoshis: BigInt(output.satoshis), + })), + sizeBytes: bitcoreTransaction.toBuffer().length, + version: txObject.version, + }; +}; + +const referenceBlockConversion = ( + bitcoreBlock: BitcoreBlock, + height: number +): ChaingraphBlock => { + const bitcoreBlockHeader = bitcoreBlock.header.toObject(); + return { + bits: bitcoreBlockHeader.bits, + hash: bitcoreBlockHeader.hash, + height, + merkleRoot: bitcoreBlockHeader.merkleRoot.toString('hex'), + nonce: bitcoreBlockHeader.nonce, + previousBlockHash: bitcoreBlockHeader.prevHash.toString('hex'), + sizeBytes: bitcoreBlock.toBuffer().length, + timestamp: bitcoreBlockHeader.time, + transactions: bitcoreBlock.transactions.map(referenceTransactionConversion), + version: bitcoreBlockHeader.version, + }; +}; + +const realTransactions = [halTxRaw, halTxSpentRaw, chipnetCashTokensTx]; + +/** + * A block containing real mainnet and chipnet (CashTokens) transactions. + */ +const realTransactionsBlock = () => { + const genesis = Block.fromString(genesisBlockRaw); + const coinbase = genesis.transactions[0]!.toObject(); + return Block.fromObject({ + header: { + bits: 0x1d00ffff, + merkleRoot: + '0000000000000000000000000000000000000000000000000000000000000000', + nonce: 1, + prevHash: genesis.header.hash, + time: 1231006506, + version: 1, + }, + transactions: [ + coinbase, + ...realTransactions.map((raw) => new Transaction(raw).toObject()), + ], + }); +}; + +const encodedTestBlocks = () => [ + Buffer.from(genesisBlockRaw, 'hex'), + Buffer.from(testnetGenesisBlockRaw, 'hex'), + Buffer.from(realTransactionsBlock().toBuffer()), + ...generateMockchain({ + length: 50, + previousBlockHash: Block.fromString(genesisBlockRaw).header.hash, + }).map((block) => Buffer.from(block.toBuffer())), +]; + +test('decodeBlockRetainingWireBytes: decodes like Block.fromBuffer', (t) => { + encodedTestBlocks().forEach((encoded) => { + const original = Buffer.from(encoded); + const block = decodeBlockRetainingWireBytes(encoded); + t.deepEqual(encoded, original, 'does not modify the encoded block'); + t.deepEqual(block.toObject(), Block.fromBuffer(original).toObject()); + t.deepEqual( + decodeBlockRetainingWireBytes(encoded).toObject(), + Block.fromBuffer(encoded).toObject() + ); + }); +}); + +test('bitcoreBlockToChaingraphBlock: matches the re-serializing implementation', (t) => { + /** + * Note: bitcore's `Block.fromBuffer` modifies the provided buffer (token + * categories are reversed in place), so each decoding gets a fresh copy. + */ + encodedTestBlocks().forEach((encoded, height) => { + const expected = referenceBlockConversion( + Block.fromBuffer(Buffer.from(encoded)), + height + ); + t.is(expected.sizeBytes, encoded.length); + t.deepEqual( + bitcoreBlockToChaingraphBlock( + decodeBlockRetainingWireBytes(Buffer.from(encoded)), + height + ), + expected, + 'with wire bytes' + ); + t.deepEqual( + bitcoreBlockToChaingraphBlock( + messages.Block.fromBuffer(Buffer.from(encoded)).block, + height + ), + expected, + 'decoded by Messages' + ); + t.deepEqual( + bitcoreBlockToChaingraphBlock( + Block.fromBuffer(Buffer.from(encoded)), + height + ), + expected, + 'without wire bytes' + ); + }); +}); + +test('bitcoreBlockToChaingraphBlock: real transaction hashes', (t) => { + const block = bitcoreBlockToChaingraphBlock( + decodeBlockRetainingWireBytes( + Buffer.from(realTransactionsBlock().toBuffer()) + ), + 1 + ); + t.deepEqual( + block.transactions.slice(1).map((tx) => tx.hash), + [halTxHash, halTxSpent, chipnetCashTokensTxHash] + ); + t.true(block.transactions[0]!.isCoinbase); + t.true( + block.transactions[3]!.outputs.some( + (output) => output.tokenCategory !== undefined + ) + ); +}); + +test('bitcoreTransactionToChaingraphTransaction: matches the re-serializing implementation', (t) => { + realTransactions.forEach((raw) => { + t.deepEqual( + bitcoreTransactionToChaingraphTransaction(new Transaction(raw)), + referenceTransactionConversion(new Transaction(raw)) + ); + }); +}); diff --git a/src/bitcore.ts b/src/bitcore.ts index ee0a00ff..d523e0c7 100644 --- a/src/bitcore.ts +++ b/src/bitcore.ts @@ -1,3 +1,5 @@ +import crypto, { createHash } from 'crypto'; + import type { BitcoreBlock, BitcoreTransaction, @@ -9,34 +11,157 @@ import type { ChaingraphTransaction, } from './types/chaingraph.js'; +const { internalBitcore } = bitcoreP2pCash; + +/** + * The serialized block as received from the P2P network, with the boundaries + * of each transaction within it. Used to take block/transaction sizes and + * transaction hashes directly from the wire bytes, rather than re-serializing + * each transaction (and the full block) with bitcore. + */ +interface BlockWireBytes { + /** + * The serialized block. + */ + encoded: Buffer; + /** + * The offset of each transaction within `encoded`, followed by the offset + * at which the final transaction ends (`transactions.length + 1` items). + */ + transactionBoundaries: number[]; +} + +const blockWireBytes = new WeakMap(); + +/** + * Decode a serialized block, retaining the serialized bytes and the boundaries + * of each transaction for use by {@link bitcoreBlockToChaingraphBlock}. The + * returned block is identical to the result of bitcore's `Block.fromBuffer`. + */ +export const decodeBlockRetainingWireBytes = (encoded: Buffer) => { + /** + * Bitcore decodes from a copy: its token prefix decoding reverses each token + * category in place within the buffer being read, which would otherwise + * corrupt the retained wire bytes. + */ + const reader = new internalBitcore.encoding.BufferReader( + Buffer.from(encoded) + ); + const header = internalBitcore.BlockHeader.fromBufferReader(reader); + const transactionCount = reader.readVarintNum(); + const transactions: BitcoreTransaction[] = []; + const transactionBoundaries: number[] = []; + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let index = 0; index < transactionCount; index += 1) { + transactionBoundaries.push(reader.pos); + transactions.push( + new internalBitcore.Transaction().fromBufferReader(reader) + ); + } + transactionBoundaries.push(reader.pos); + const block = new internalBitcore.Block({ header, transactions }); + blockWireBytes.set(block, { encoded, transactionBoundaries }); + return block; +}; + +/** + * A bitcore `Block` constructor which decodes blocks using + * {@link decodeBlockRetainingWireBytes}. Provided to bitcore-p2p-cash's + * `Messages` so blocks received from peers retain their wire bytes. + */ +export class BlockRetainingWireBytes extends internalBitcore.Block { + static override fromBuffer = decodeBlockRetainingWireBytes; +} + +/** + * Create the bitcore-p2p-cash `Messages` used to communicate with a peer on + * the provided network. Received blocks retain their wire bytes. + */ +export const createPeerMessages = (networkName: string) => + new bitcoreP2pCash.Messages({ + // eslint-disable-next-line @typescript-eslint/naming-convention + Block: BlockRetainingWireBytes, + network: internalBitcore.Networks.get(networkName), + }); + +/** + * Node.js' one-shot `crypto.hash` (Node.js v20.12+, v21.7+) avoids creating a + * `Hash` object per call. (Not yet included in this project's `@types/node`.) + */ +const oneShotHash = ( + crypto as unknown as { + hash?: (algorithm: string, data: Uint8Array, encoding: 'buffer') => Buffer; + } +).hash; + +const sha256 = + oneShotHash === undefined + ? (data: Uint8Array) => createHash('sha256').update(data).digest() + : (data: Uint8Array) => oneShotHash('sha256', data, 'buffer'); + +/** + * Compute the transaction hash (A.K.A. "transaction ID") of an encoded transaction: + * the double-SHA256 hash, in user-interface (reversed) byte order, hex-encoded. + */ +export const encodedTransactionToHash = (encodedTransaction: Uint8Array) => + sha256(sha256(encodedTransaction)).reverse().toString('hex'); + +const coinbaseOutpointIndex = 0xffffffff; +const hashLength = 32; +const coinbaseOutpointTransactionHash = '00'.repeat(hashLength); + +/** + * Convert a bitcore transaction to the Chaingraph format. + * @param bitcoreTransaction - the bitcore transaction + * @param encodedTransaction - the serialized transaction, if available (e.g. + * as received from the network); if not provided, the transaction is + * re-serialized + */ export const bitcoreTransactionToChaingraphTransaction = ( - bitcoreTransaction: BitcoreTransaction + bitcoreTransaction: BitcoreTransaction, + encodedTransaction: Uint8Array = bitcoreTransaction.toBuffer() ): ChaingraphTransaction => { - const txObject = bitcoreTransaction.toObject(); - const isCoinbase = bitcoreTransaction.isCoinbase(); + /** + * Fields are read directly rather than via bitcore's `Input.toObject`, which + * also decodes and stringifies each unlocking bytecode (`scriptString`). + */ + const inputs = bitcoreTransaction.inputs.map((input) => ({ + outpointIndex: input.outputIndex, + outpointTransactionHash: input.prevTxId.toString('hex'), + sequenceNumber: input.sequenceNumber, + // eslint-disable-next-line no-underscore-dangle + unlockingBytecode: input._scriptBuffer.toString('hex'), + })); + /** + * Equivalent to bitcore's `Transaction.isCoinbase`, reusing the hex-encoded + * outpoint transaction hash. + */ + const isCoinbase = + inputs.length === 1 && + inputs[0]!.outpointIndex === coinbaseOutpointIndex && + inputs[0]!.outpointTransactionHash === coinbaseOutpointTransactionHash; return { - hash: txObject.hash, - inputs: txObject.inputs.map((input) => ({ - outpointIndex: input.outputIndex, - outpointTransactionHash: input.prevTxId, - sequenceNumber: input.sequenceNumber, - unlockingBytecode: input.script, - })), + hash: encodedTransactionToHash(encodedTransaction), + inputs, isCoinbase, - locktime: txObject.nLockTime, - outputs: txObject.outputs.map((output) => ({ - fungibleTokenAmount: - output.tokenData === undefined - ? undefined - : BigInt(output.tokenData.amount), - lockingBytecode: output.script, - nonfungibleTokenCapability: output.tokenData?.nft?.capability, - nonfungibleTokenCommitment: output.tokenData?.nft?.commitment, - tokenCategory: output.tokenData?.category, - valueSatoshis: BigInt(output.satoshis), - })), - sizeBytes: bitcoreTransaction.toBuffer().length, - version: txObject.version, + locktime: bitcoreTransaction.nLockTime, + outputs: bitcoreTransaction.outputs.map((output) => { + const { tokenData } = output; + return { + fungibleTokenAmount: + tokenData === undefined + ? undefined + : BigInt(tokenData.amount.toString()), + // eslint-disable-next-line no-underscore-dangle + lockingBytecode: output._scriptBuffer.toString('hex'), + nonfungibleTokenCapability: tokenData?.nft?.capability, + nonfungibleTokenCommitment: tokenData?.nft?.commitment, + tokenCategory: tokenData?.category, + valueSatoshis: BigInt(output.satoshis), + }; + }), + sizeBytes: encodedTransaction.length, + version: bitcoreTransaction.version, }; }; @@ -45,6 +170,21 @@ export const bitcoreBlockToChaingraphBlock = ( height: number ): ChaingraphBlock => { const bitcoreBlockHeader = bitcoreBlock.header.toObject(); + const wireBytes = blockWireBytes.get(bitcoreBlock); + const transactions = + wireBytes === undefined + ? bitcoreBlock.transactions.map((transaction) => + bitcoreTransactionToChaingraphTransaction(transaction) + ) + : bitcoreBlock.transactions.map((transaction, index) => + bitcoreTransactionToChaingraphTransaction( + transaction, + wireBytes.encoded.subarray( + wireBytes.transactionBoundaries[index], + wireBytes.transactionBoundaries[index + 1] + ) + ) + ); return { bits: bitcoreBlockHeader.bits, hash: bitcoreBlockHeader.hash, @@ -52,13 +192,19 @@ export const bitcoreBlockToChaingraphBlock = ( merkleRoot: bitcoreBlockHeader.merkleRoot.toString('hex'), nonce: bitcoreBlockHeader.nonce, previousBlockHash: bitcoreBlockHeader.prevHash.toString('hex'), - sizeBytes: bitcoreBlock.toBuffer().length, + sizeBytes: + wireBytes === undefined + ? bitcoreBlock.toBuffer().length + : wireBytes.transactionBoundaries[ + wireBytes.transactionBoundaries.length - 1 + ]!, timestamp: bitcoreBlockHeader.time, - transactions: bitcoreBlock.transactions.map( - bitcoreTransactionToChaingraphTransaction - ), + transactions, version: bitcoreBlockHeader.version, }; }; -export const messages = new bitcoreP2pCash.Messages(); +export const messages = new bitcoreP2pCash.Messages({ + // eslint-disable-next-line @typescript-eslint/naming-convention + Block: BlockRetainingWireBytes, +}); diff --git a/src/types/bitcore-lib-cash.d.ts b/src/types/bitcore-lib-cash.d.ts index fe1f9cf5..e9bb8878 100644 --- a/src/types/bitcore-lib-cash.d.ts +++ b/src/types/bitcore-lib-cash.d.ts @@ -50,6 +50,16 @@ declare module 'bitcore-lib-cash' { version: number; } + export namespace encoding { + class BufferReader { + pos: number; + + readVarintNum: () => number; + + constructor(buffer: Buffer); + } + } + export interface BitcoreBlockHeader { hash: string; prevHash: Buffer; @@ -60,9 +70,17 @@ declare module 'bitcore-lib-cash' { toHex: () => string; } + export namespace BlockHeader { + function fromBufferReader( + reader: encoding.BufferReader + ): BitcoreBlockHeader; + } + export class Transaction { toObject: () => BitcoreTransactionObject; + fromBufferReader: (reader: encoding.BufferReader) => this; + toBuffer: () => Uint8Array; isCoinbase: () => boolean; @@ -70,10 +88,12 @@ declare module 'bitcore-lib-cash' { hash: string; inputs: { - prevTxId: string; + prevTxId: Buffer; outputIndex: number; sequenceNumber: number; script: BitcoreScript; + _scriptBuffer: Buffer; + toObject: () => BitcoreTransactionObject['inputs'][number]; }[]; nLockTime: number; @@ -81,8 +101,13 @@ declare module 'bitcore-lib-cash' { outputs: { satoshis: number; script: BitcoreScript; + _scriptBuffer: Buffer; + toObject: () => BitcoreTransactionObject['outputs'][number]; tokenData?: { - amount: string; + /** + * A bn.js `BN` + */ + amount: { toString: () => string }; category: string; nft?: { capability: 'minting' | 'mutable' | 'none'; @@ -93,19 +118,28 @@ declare module 'bitcore-lib-cash' { version: number; - constructor(hexOrObject: BitcoreTransactionObjectOptions | string); + constructor(hexOrObject?: BitcoreTransactionObjectOptions | string); } export class Block { + static fromBuffer: (encoded: Buffer) => Block; + static fromObject: (contents: unknown) => Block; static fromString: (raw: string) => Block; toBuffer: () => Uint8Array; + toObject: () => unknown; + header: BitcoreBlockHeader; transactions: Transaction[]; + + constructor(contents: { + header: BitcoreBlockHeader; + transactions: Transaction[]; + }); } interface Network { diff --git a/src/types/bitcore-p2p-cash.d.ts b/src/types/bitcore-p2p-cash.d.ts index e339c271..16172db5 100644 --- a/src/types/bitcore-p2p-cash.d.ts +++ b/src/types/bitcore-p2p-cash.d.ts @@ -23,6 +23,12 @@ declare module '@chaingraph/bitcore-p2p-cash' { Block: { fromBuffer: (rawBlock: Buffer) => BlockMessage; }; + + constructor(options?: { + Block?: typeof Block; + network?: ReturnType; + Transaction?: typeof Transaction; + }); } export const enum BitcoreInventoryType { @@ -132,6 +138,7 @@ declare module '@chaingraph/bitcore-p2p-cash' { constructor(config: { host?: string; + messages?: Messages; network?: string; port?: number; relay?: boolean; From c620babca7105070fac8d5dc1f0687dadc88e94d Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 21 Oct 2026 09:27:30 +0100 Subject: [PATCH 61/63] perf(agent): skip already-audited blocks in startup repair scan; linear SyncState (#12) The startup incomplete-block repair scan records, per node, the highest block internal_id it has fully audited and skips those blocks on later starts (falling back to the full scan when any node has no marker). SyncState tracking becomes linear. Adds e2e coverage for markers, the bounded per-node scan and the full-audit fallback. Co-Authored-By: Claude Opus 5.5 --- charts/chaingraph/values.yaml | 2 +- defaults.env | 4 +- .../down.sql | 1 + .../up.sql | 11 + src/agent.ts | 110 +++++++++- src/components/sync-state.spec.ts | 36 ++++ src/components/sync-state.ts | 56 +++-- src/db.ts | 162 +++++++++++++- src/e2e/e2e.spec.ts | 201 ++++++++++++++++++ 9 files changed, 544 insertions(+), 39 deletions(-) create mode 100644 images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/down.sql create mode 100644 images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/up.sql diff --git a/charts/chaingraph/values.yaml b/charts/chaingraph/values.yaml index 68616cee..941e8193 100644 --- a/charts/chaingraph/values.yaml +++ b/charts/chaingraph/values.yaml @@ -25,7 +25,7 @@ agent: # For best performance, this should be around `maxConnections * maximum block size`, while leaving enough memory available to the host machine. If left unset (recommended), Chaingraph will measure free memory at startup and attempt to select a reasonable value. blockBufferTargetSizeMb: '' # Self-healing for https://github.com/bitauth/chaingraph/issues/74: after initial sync, the agent scans saved blocks in batches to find blocks whose linked transactions don't match the saved block size, then repairs those blocks by re-requesting them from trusted nodes. - # Set to 0 to disable the startup repair task. + # Set to 0 to disable the startup repair task. Completed scans are recorded per node in the incomplete_block_audit table, so later startups only audit blocks saved since the last completed scan; delete its rows to force a full re-audit. repairBlocksBatchSize: 10000 # A mapping of network magic bytes to hex-encoded genesis blocks. # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated. diff --git a/defaults.env b/defaults.env index 35f826ee..5bbfb237 100644 --- a/defaults.env +++ b/defaults.env @@ -20,7 +20,9 @@ CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB= # initial sync, the agent scans saved blocks in batches to find blocks whose # linked transactions don't match the saved block size, then repairs those # blocks by re-requesting them from trusted nodes. Set to 0 to disable the -# startup repair task. +# startup repair task. Completed scans are recorded per node in the +# incomplete_block_audit table, so later startups only audit blocks saved since +# the last completed scan; delete its rows to force a full re-audit. CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE=10000 # BCHN expires transactions from the mempool after 336 hours but does not announce diff --git a/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/down.sql b/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/down.sql new file mode 100644 index 00000000..0aaf0256 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS incomplete_block_audit; diff --git a/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/up.sql b/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/up.sql new file mode 100644 index 00000000..381894e5 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/up.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS incomplete_block_audit ( + node_internal_id integer NOT NULL, + audited_through_block_internal_id bigint NOT NULL, + completed_at timestamp without time zone DEFAULT now() NOT NULL, + CONSTRAINT incomplete_block_audit_pkey PRIMARY KEY (node_internal_id), + CONSTRAINT incomplete_block_audit_node_internal_id_fkey FOREIGN KEY (node_internal_id) REFERENCES node(internal_id) ON UPDATE RESTRICT ON DELETE CASCADE +); +COMMENT ON TABLE incomplete_block_audit IS 'Internal agent bookkeeping for the incomplete block repair scan (https://github.com/bitauth/chaingraph/issues/74). Each row records that every block accepted by the node with internal_id at or below audited_through_block_internal_id has been audited, so the agent skips those blocks on later startups. Delete rows to force a full re-audit.'; +COMMENT ON COLUMN incomplete_block_audit.node_internal_id IS 'The internal_id of the node whose accepted blocks were audited.'; +COMMENT ON COLUMN incomplete_block_audit.audited_through_block_internal_id IS 'The highest block internal_id which existed when the completed audit began.'; +COMMENT ON COLUMN incomplete_block_audit.completed_at IS 'The time at which the audit completed.'; diff --git a/src/agent.ts b/src/agent.ts index b2df6738..9d8d6f1a 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -47,6 +47,7 @@ import { createIndexes, expireMempoolTransaction, getAllKnownBlockHashes, + getIncompleteBlockAuditState, getIncompleteBlocks, getIndexCreationProgress, getMempoolTransactionsExpiringBefore, @@ -59,9 +60,14 @@ import { registerTrustedNodeWithDb, removeStaleBlocksForNode, saveBlock, + saveIncompleteBlockAudit, saveTransactionForNodes, } from './db.js'; -import type { ExpiringMempoolTransaction, IncompleteBlock } from './db.js'; +import type { + ExpiringMempoolTransaction, + IncompleteBlock, + IncompleteBlockAuditState, +} from './db.js'; import { describePostgresError, isPostgresConnectionError, @@ -382,6 +388,20 @@ export class Agent { completedIncompleteBlockRepairScan = false; + /** + * State of the in-progress incomplete block repair scan: the audit markers + * loaded when it began, and the nodes which were registered for every batch + * (only those nodes' markers are advanced when the scan completes). + */ + incompleteBlockAuditPass: + | { + auditState: IncompleteBlockAuditState; + auditedBlockCount: number; + nodeInternalIdsInEveryBatch: number[]; + startedAt: number; + } + | undefined; + /** * The next second after which to log another warning that one or more nodes * are unresponsive. @@ -1336,16 +1356,98 @@ export class Agent { }, Promise.resolve()); } + /** + * Begin (or continue) an incomplete block audit pass. Blocks which a + * previous, completed pass already audited for a node are skipped for that + * node, so restarting a synced agent doesn't re-read every saved + * transaction (~51 GB on an 831k-block mainnet database). + */ + async getIncompleteBlockAuditPass(nodeInternalIds: number[]) { + if (this.incompleteBlockAuditPass !== undefined) { + return this.incompleteBlockAuditPass; + } + const auditState = await getIncompleteBlockAuditState(nodeInternalIds); + const auditPass = { + auditState, + auditedBlockCount: 0, + nodeInternalIdsInEveryBatch: nodeInternalIds, + startedAt: Date.now(), + }; + this.incompleteBlockAuditPass = auditPass; + if (!auditState.auditTableExists) { + this.logger.warn( + 'Agent: database schema has no incomplete_block_audit table, so the incomplete block repair scan will audit every saved block on each startup. Update the Hasura image to apply migrations.' + ); + return auditPass; + } + const markers = nodeInternalIds.map( + (id) => + `${id}: ${auditState.auditedThroughBlockInternalIds.get(id) ?? 'none'}` + ); + this.logger.info( + `Agent: starting incomplete block repair scan; skipping blocks already audited per node (node internal_id: audited through block internal_id) – ${markers.join( + ', ' + )}.` + ); + return auditPass; + } + + async completeIncompleteBlockAuditPass() { + const auditPass = this.incompleteBlockAuditPass; + this.incompleteBlockAuditPass = undefined; + if (auditPass === undefined) { + return; + } + const { auditState, auditedBlockCount, nodeInternalIdsInEveryBatch } = + auditPass; + const durationSeconds = ( + (Date.now() - auditPass.startedAt) / + msPerSecond + ).toFixed(1); + this.logger.info( + `Agent: incomplete block repair scan audited ${auditedBlockCount.toLocaleString()} block(s) in ${durationSeconds} seconds.` + ); + if (!auditState.auditTableExists) { + return; + } + await saveIncompleteBlockAudit({ + auditedThroughBlockInternalId: auditState.maxBlockInternalId, + nodeInternalIds: nodeInternalIdsInEveryBatch, + }) + .then(() => { + this.logger.debug( + `Agent: recorded incomplete block audit through block internal_id ${ + auditState.maxBlockInternalId + } for node internal_id(s): ${nodeInternalIdsInEveryBatch.join(', ')}.` + ); + }) + .catch((err) => { + this.logger.error( + err, + 'Agent: failed to record incomplete block audit; the next startup will audit these blocks again.' + ); + }); + } + async repairIncompleteBlocksOnce() { const { finalHeight, heightLowerBound, heightUpperBound } = this.getIncompleteBlockRepairRange(); + const nodeInternalIds = this.getRegisteredNodeInternalIds(); + const auditPass = await this.getIncompleteBlockAuditPass(nodeInternalIds); + auditPass.nodeInternalIdsInEveryBatch = + auditPass.nodeInternalIdsInEveryBatch.filter((id) => + nodeInternalIds.includes(id) + ); const scanStartTime = Date.now(); const { incompleteBlocks, scannedBlockCount } = await getIncompleteBlocks({ + auditedThroughBlockInternalIds: nodeInternalIds.map( + (id) => auditPass.auditState.auditedThroughBlockInternalIds.get(id) ?? 0 + ), excludedBlockHashes: [], heightLowerBound, heightUpperBound, limit: incompleteBlockRepairBatchSize, - nodeInternalIds: this.getRegisteredNodeInternalIds(), + nodeInternalIds, }); const scanDurationMs = Date.now() - scanStartTime; const scanRate = @@ -1354,6 +1456,10 @@ export class Agent { : Math.round((scannedBlockCount / scanDurationMs) * msPerSecond); const scanPerformanceLog = `scanned ${scannedBlockCount.toLocaleString()} block(s) in ${scanDurationMs.toLocaleString()}ms (${scanRate.toLocaleString()} blocks/s)`; if (incompleteBlocks.length === 0) { + auditPass.auditedBlockCount += scannedBlockCount; + if (heightUpperBound === finalHeight) { + await this.completeIncompleteBlockAuditPass(); + } this.updateIncompleteBlockRepairProgress({ finalHeight, heightLowerBound, diff --git a/src/components/sync-state.spec.ts b/src/components/sync-state.spec.ts index 4f5fb3cc..0b6e5b45 100644 --- a/src/components/sync-state.spec.ts +++ b/src/components/sync-state.spec.ts @@ -86,3 +86,39 @@ test('SyncState scales when marking many contiguous heights as pending', (t) => `Expected marking contiguous pending heights to complete in less than ${maximumDurationMs}ms; took ${durationMs}ms.` ); }); + +test('SyncState scales when marking many pending heights as synced', (t) => { + const state = new SyncState({ + additionalSyncedHeights: [], + fullySyncedUpToHeight: 0, + pendingSyncOfHeights: [], + }); + /* + * Mirrors `catchUpViaHeaders` accepting a long run of already-known blocks + * (e.g. a new network sharing history with an existing one): every height is + * marked pending, then every height is marked synced in ascending order. With + * array-backed state this was quadratic (~7s at 300k heights). + */ + const acceptedThroughHeight = 300_000; + const maximumDurationMs = 1_000; + + const startedAt = Date.now(); + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let height = 1; height <= acceptedThroughHeight; height += 1) { + state.markHeightAsPendingSync(height); + } + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let height = 1; height <= acceptedThroughHeight; height += 1) { + state.markHeightAsSynced(height, 'caught-up'); + } + const durationMs = Date.now() - startedAt; + + t.deepEqual(state.fullySyncedUpToHeight, acceptedThroughHeight); + t.deepEqual(state.getPendingSyncHeight(), acceptedThroughHeight); + t.deepEqual(state.pendingSyncOfHeights, []); + t.deepEqual(state.additionalSyncedHeights, []); + t.true( + durationMs < maximumDurationMs, + `Expected marking pending heights as synced to complete in less than ${maximumDurationMs}ms; took ${durationMs}ms.` + ); +}); diff --git a/src/components/sync-state.ts b/src/components/sync-state.ts index 58f6e5c8..6a40f660 100644 --- a/src/components/sync-state.ts +++ b/src/components/sync-state.ts @@ -4,15 +4,6 @@ export interface InitialSyncState { additionalSyncedHeights: number[]; } -const removeValueIfPresent = (array: number[], value: number) => { - const index = array.indexOf(value); - if (index !== -1) { - array.splice(index, 1); - return true; - } - return false; -}; - /** * A simple data structure to keep track of the known database syncing progress * for a particular node. @@ -20,10 +11,6 @@ const removeValueIfPresent = (array: number[], value: number) => { export class SyncState { fullySyncedUpToHeight: number; - pendingSyncOfHeights: number[]; - - additionalSyncedHeights: number[]; - /** * The block time claimed by the latest synced block. Chaingraph assumes the * trusted node is itself not fully-synced if this time is older than 2 hours @@ -56,10 +43,10 @@ export class SyncState { constructor(initialState: InitialSyncState) { this.fullySyncedUpToHeight = initialState.fullySyncedUpToHeight; - this.pendingSyncOfHeights = initialState.pendingSyncOfHeights.slice(); - this.additionalSyncedHeights = initialState.additionalSyncedHeights.slice(); - this.pendingSyncHeights = new Set(this.pendingSyncOfHeights); - this.additionalSyncedHeightSet = new Set(this.additionalSyncedHeights); + this.pendingSyncHeights = new Set(initialState.pendingSyncOfHeights); + this.additionalSyncedHeightSet = new Set( + initialState.additionalSyncedHeights + ); this.pendingSyncHeight = this.computePendingSyncHeightFrom( this.fullySyncedUpToHeight ); @@ -67,6 +54,24 @@ export class SyncState { initialState.fullySyncedUpToHeight > 0 ? 'caught-up' : undefined; } + /** + * Heights currently pending sync, in the order they were marked. (Derived + * from a `Set`: removing a height is O(1), so marking many heights as synced + * – e.g. after `catchUpViaHeaders` accepts a long run of known blocks – is + * linear rather than quadratic.) + */ + get pendingSyncOfHeights() { + return [...this.pendingSyncHeights]; + } + + /** + * Heights above `fullySyncedUpToHeight` which have been synced, in the order + * they were marked. + */ + get additionalSyncedHeights() { + return [...this.additionalSyncedHeightSet]; + } + // eslint-disable-next-line complexity markHeightAsSynced(height: number, claimedBlockTime: Date | 'caught-up') { if (claimedBlockTime === 'caught-up') { @@ -82,18 +87,14 @@ export class SyncState { this.fullySyncedUpToHeight < height && !this.additionalSyncedHeightSet.has(height) ) { - this.additionalSyncedHeights.push(height); this.additionalSyncedHeightSet.add(height); - if (this.pendingSyncHeights.delete(height)) { - removeValueIfPresent(this.pendingSyncOfHeights, height); - } + this.pendingSyncHeights.delete(height); } // eslint-disable-next-line functional/no-let let nextHeight = this.fullySyncedUpToHeight + 1; // eslint-disable-next-line functional/no-loop-statement while (this.additionalSyncedHeightSet.delete(nextHeight)) { - removeValueIfPresent(this.additionalSyncedHeights, nextHeight); this.fullySyncedUpToHeight = nextHeight; nextHeight += 1; } @@ -106,7 +107,6 @@ export class SyncState { !this.pendingSyncHeights.has(height) && !this.additionalSyncedHeightSet.has(height) ) { - this.pendingSyncOfHeights.push(height); this.pendingSyncHeights.add(height); this.updatePendingSyncHeight(); } @@ -125,14 +125,12 @@ export class SyncState { ? this.fullySyncedUpToHeight : height - 1; - this.pendingSyncOfHeights = this.pendingSyncOfHeights.filter( - (pending) => pending < height + this.pendingSyncHeights = new Set( + this.pendingSyncOfHeights.filter((pending) => pending < height) ); - this.additionalSyncedHeights = this.additionalSyncedHeights.filter( - (completed) => completed < height + this.additionalSyncedHeightSet = new Set( + this.additionalSyncedHeights.filter((completed) => completed < height) ); - this.pendingSyncHeights = new Set(this.pendingSyncOfHeights); - this.additionalSyncedHeightSet = new Set(this.additionalSyncedHeights); this.resetPendingSyncHeight(); } diff --git a/src/db.ts b/src/db.ts index b6bbb8ad..d976c615 100644 --- a/src/db.ts +++ b/src/db.ts @@ -128,14 +128,27 @@ export interface ArchivedMempoolTransaction { * Find blocks for which the locally saved block_transaction rows don't sum to * the block's saved byte size. This avoids the SQL block encoder so it can * detect incomplete blocks even if encoder functions have bugs (e.g. #75). + * + * Only blocks accepted by one of `nodeInternalIds` are checked, and for each + * node only blocks with an `internal_id` above the matching entry of + * `auditedThroughBlockInternalIds` (0 if the node has never completed an + * audit). Summing `transaction.size_bytes` reads every linked transaction row, + * so on a large, already-audited database this filter is what keeps the scan + * from re-reading the whole `transaction` table on every startup. */ export const getIncompleteBlocks = async ({ + auditedThroughBlockInternalIds, heightLowerBound, heightUpperBound, limit, nodeInternalIds, excludedBlockHashes, }: { + /** + * Aligned with `nodeInternalIds`; defaults to 0 (audit every block) for + * each node. + */ + auditedThroughBlockInternalIds?: number[]; excludedBlockHashes: string[]; heightLowerBound: number; heightUpperBound: number; @@ -145,6 +158,37 @@ export const getIncompleteBlocks = async ({ if (nodeInternalIds.length === 0) { return { incompleteBlocks: [], scannedBlockCount: 0 }; } + // cspell:ignore unnest + const auditedThrough = nodeInternalIds.map( + (_, index) => auditedThroughBlockInternalIds?.[index] ?? 0 + ); + /* + * If any node has never completed an audit, every block it accepted must be + * audited anyway, so use the original (unbounded) filter: on a full audit + * it is measurably cheaper than the per-node join (measured on an 831k-block + * mainnet checkpoint: 413 s vs 552 s for the same bytes read). The per-node bounded filter is used + * only when every node has a marker. + */ + const minimumAuditedThrough = Math.min(...auditedThrough); + const auditEveryBlock = minimumAuditedThrough === 0; + const acceptedByNodeFilter = auditEveryBlock + ? /* sql */ ` + AND EXISTS ( + SELECT 1 FROM node_block + WHERE node_block.block_internal_id = block.internal_id + AND node_block.node_internal_id = ANY($1::integer[]) + )` + : /* sql */ ` + AND block.internal_id > $7 + AND EXISTS ( + SELECT 1 FROM node_block + INNER JOIN unnest($1::integer[], $6::bigint[]) + AS audited_node (node_internal_id, audited_through_block_internal_id) + ON audited_node.node_internal_id = node_block.node_internal_id + WHERE node_block.block_internal_id = block.internal_id + AND block.internal_id > + audited_node.audited_through_block_internal_id + )`; const client = await pool.connect(); // eslint-disable-next-line functional/no-try-statement try { @@ -176,12 +220,7 @@ WITH linked_transactions AS ( ON transaction.internal_id = block_transaction.transaction_internal_id WHERE block.height >= $2 AND block.height < $3 - AND NOT (encode(block.hash, 'hex') = ANY($5::text[])) - AND EXISTS ( - SELECT 1 FROM node_block - WHERE node_block.block_internal_id = block.internal_id - AND node_block.node_internal_id = ANY($1::integer[]) - ) + AND NOT (encode(block.hash, 'hex') = ANY($5::text[]))${acceptedByNodeFilter} GROUP BY block.internal_id ), linked_block_sizes AS ( @@ -237,6 +276,7 @@ SELECT heightUpperBound, limit, excludedBlockHashes, + ...(auditEveryBlock ? [] : [auditedThrough, minimumAuditedThrough]), ] ); const scan = incompleteBlockScan.rows[0]!; @@ -255,6 +295,116 @@ SELECT } }; +export interface IncompleteBlockAuditState { + /** + * False if the database schema predates the `incomplete_block_audit` table; + * the agent then audits every block on each startup (and saves nothing). + */ + auditTableExists: boolean; + /** + * Per node internal_id, the block internal_id through which a previous audit + * completed. + */ + auditedThroughBlockInternalIds: Map; + /** + * The highest block internal_id currently saved (0 if there are no blocks). + * Recorded as the new audit marker once this audit completes. + */ + maxBlockInternalId: number; +} + +/** + * Load the incomplete block audit markers for the provided nodes, along with + * the current highest block internal_id. + */ +export const getIncompleteBlockAuditState = async ( + nodeInternalIds: number[] +): Promise => { + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const { auditTableExists, maxBlockInternalId } = ( + await client.query<{ + auditTableExists: boolean; + maxBlockInternalId: string; + }>(/* sql */ ` +SELECT + to_regclass('incomplete_block_audit') IS NOT NULL AS "auditTableExists", + COALESCE((SELECT MAX(internal_id) FROM block), 0)::bigint AS "maxBlockInternalId"; +`) + ).rows[0]!; + if (!auditTableExists) { + return { + auditTableExists, + auditedThroughBlockInternalIds: new Map(), + maxBlockInternalId: Number(maxBlockInternalId), + }; + } + const markers = await client.query<{ + auditedThroughBlockInternalId: string; + nodeInternalId: number; + }>( + /* sql */ ` +SELECT node_internal_id AS "nodeInternalId", + audited_through_block_internal_id AS "auditedThroughBlockInternalId" + FROM incomplete_block_audit + WHERE node_internal_id = ANY($1::integer[]); +`, + [nodeInternalIds] + ); + return { + auditTableExists, + auditedThroughBlockInternalIds: new Map( + markers.rows.map((row) => [ + row.nodeInternalId, + Number(row.auditedThroughBlockInternalId), + ]) + ), + maxBlockInternalId: Number(maxBlockInternalId), + }; + } finally { + client.release(); + } +}; + +/** + * Record that every block accepted by each of `nodeInternalIds` with an + * internal_id at or below `auditedThroughBlockInternalId` has been audited. + * Markers never move backwards. + */ +export const saveIncompleteBlockAudit = async ({ + auditedThroughBlockInternalId, + nodeInternalIds, +}: { + auditedThroughBlockInternalId: number; + nodeInternalIds: number[]; +}) => { + if (nodeInternalIds.length === 0) { + return; + } + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query( + /* sql */ ` +INSERT INTO incomplete_block_audit + (node_internal_id, audited_through_block_internal_id, completed_at) + SELECT node_internal_id, $2::bigint, now() + FROM unnest($1::integer[]) AS audited_node (node_internal_id) + ON CONFLICT (node_internal_id) DO UPDATE + SET audited_through_block_internal_id = GREATEST( + incomplete_block_audit.audited_through_block_internal_id, + EXCLUDED.audited_through_block_internal_id + ), + completed_at = EXCLUDED.completed_at; +`, + [nodeInternalIds, auditedThroughBlockInternalId] + ); + } finally { + client.release(); + } +}; + /** * Find node_transaction rows which will expire before the provided timestamp. */ diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 0acef632..9957d560 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -28,6 +28,7 @@ import got from 'got'; import pg from 'pg'; import { indexDefinitions } from '../components/db-utils.js'; +import type * as DbModule from '../db.js'; import type { ChaingraphTransaction } from '../types/chaingraph.js'; import { chaingraphE2eLogPath, logger } from './e2e.spec.logging.helper.js'; @@ -87,6 +88,7 @@ const dbUpMigrationPaths = [ migration( 'default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql' ), + migration('default/1791200000000_add_incomplete_block_audit/up.sql'), ]; const chaingraphInternalApiPort = '3201'; @@ -919,6 +921,7 @@ test.serial('[e2e] creates expected indexes after initial sync', async (t) => { 'block_internal_id_key', 'block_pkey', 'block_transaction_pkey', + 'incomplete_block_audit_pkey', 'input_pkey', 'node_block_history_pkey', 'node_block_pkey', @@ -2632,6 +2635,204 @@ test.serial( } ); +const getIncompleteBlockAuditMarkers = async () => + ( + await client.query<{ + auditedThroughBlockInternalId: string; + nodeInternalId: number; + nodeName: string; + }>(/* sql */ ` + SELECT node.name AS "nodeName", + node.internal_id AS "nodeInternalId", + incomplete_block_audit.audited_through_block_internal_id + AS "auditedThroughBlockInternalId" + FROM incomplete_block_audit + INNER JOIN node + ON node.internal_id = incomplete_block_audit.node_internal_id + ORDER BY node.name; + `) + ).rows; + +const waitForIncompleteBlockAuditMarker = async ( + nodeName: string, + remainingAttempts = blockRepairPollingAttempts +): Promise>> => { + const markers = await getIncompleteBlockAuditMarkers(); + if ( + remainingAttempts <= 0 || + markers.some((marker) => marker.nodeName === nodeName) + ) { + return markers; + } + await sleep(blockRepairPollingIntervalMs); + return waitForIncompleteBlockAuditMarker(nodeName, remainingAttempts - 1); +}; + +test.serial( + '[e2e] records incomplete block audit markers and skips audited blocks on restart', + async (t) => { + t.timeout(oneMinute); + const markers = await waitForIncompleteBlockAuditMarker('node4'); + const markersByNode = new Map( + markers.map((marker) => [marker.nodeName, marker]) + ); + ['node1', 'node2', 'node4'].forEach((nodeName) => { + t.true(markersByNode.has(nodeName), `missing marker for ${nodeName}`); + }); + const maxBlockInternalId = Number( + ( + await client.query<{ max: string }>( + /* sql */ `SELECT MAX(internal_id) AS max FROM block;` + ) + ).rows[0]!.max + ); + markers.forEach((marker) => { + const auditedThrough = Number(marker.auditedThroughBlockInternalId); + t.true(auditedThrough > 0 && auditedThrough <= maxBlockInternalId); + }); + /* + * The previous agent process completed a scan for node1 and node2, so this + * process loaded their markers (and skipped their audited blocks); node4 is + * new, so all of its blocks were audited (which repaired the incomplete + * historical block above). + */ + t.regex( + stdoutBuffer, + new RegExp( + `starting incomplete block repair scan;[^\\n]*${ + markersByNode.get('node1')!.nodeInternalId + }: \\d+[^\\n]*${markersByNode.get('node4')!.nodeInternalId}: none`, + 'u' + ) + ); + } +); + +test.serial( + '[e2e] getIncompleteBlocks skips blocks already audited for a node', + async (t) => { + const originalPostgresConnectionString = + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + postgresE2eConnectionStringTestDb; + /* + * An earlier test ends the shared `db.js` pool, so load a fresh module + * instance (with its own pool) for this test. + */ + const freshDbModuleUrl = `../db.js?incomplete-block-audit=${Date.now()}`; + const { getIncompleteBlocks, pool: freshDbPool } = (await import( + freshDbModuleUrl + )) as typeof DbModule; + const markersByNode = new Map( + (await getIncompleteBlockAuditMarkers()).map((marker) => [ + marker.nodeName, + marker, + ]) + ); + const auditedNode1 = markersByNode.get('node1')!; + const auditedNode4 = markersByNode.get('node4')!; + const removedRow = ( + await client.query<{ + blockInternalId: string; + transactionIndex: number; + transactionInternalId: string; + }>( + /* sql */ ` + DELETE FROM block_transaction + USING block + WHERE block.internal_id = block_transaction.block_internal_id + AND block.hash = $1 + AND block_transaction.transaction_index = $2 + RETURNING block_transaction.block_internal_id AS "blockInternalId", + block_transaction.transaction_internal_id + AS "transactionInternalId", + block_transaction.transaction_index AS "transactionIndex"; + `, + [hexToBin(historicalRepairBlockHash), historicalRepairTransactionIndex] + ) + ).rows[0]!; + // eslint-disable-next-line functional/no-try-statement + try { + t.true( + Number(removedRow.blockInternalId) <= + Number(auditedNode1.auditedThroughBlockInternalId) + ); + const height = historicalRepairTipIndex + splitHeight + 1; + const scan = async ( + nodeInternalIds: number[], + auditedThroughBlockInternalIds?: number[] + ) => + getIncompleteBlocks({ + auditedThroughBlockInternalIds, + excludedBlockHashes: [], + heightLowerBound: height, + heightUpperBound: height + 1, + limit: 10, + nodeInternalIds, + }); + t.deepEqual( + await scan( + [auditedNode1.nodeInternalId], + [Number(auditedNode1.auditedThroughBlockInternalId)] + ), + { incompleteBlocks: [], scannedBlockCount: 0 } + ); + const unaudited = await scan([auditedNode1.nodeInternalId]); + t.deepEqual(unaudited.scannedBlockCount, 1); + t.deepEqual( + unaudited.incompleteBlocks.map((block) => block.hash), + [historicalRepairBlockHash] + ); + const oneNodeUnaudited = await scan( + [auditedNode1.nodeInternalId, auditedNode4.nodeInternalId], + [Number(auditedNode1.auditedThroughBlockInternalId), 0] + ); + t.deepEqual( + oneNodeUnaudited.incompleteBlocks.map((block) => block.hash), + [historicalRepairBlockHash] + ); + /* + * All markers positive (the bounded per-node join, not the full-audit + * fallback): node4's marker is below the block, so the block is still + * returned although node1 has already audited it. + */ + const blockInternalId = Number(removedRow.blockInternalId); + t.true(blockInternalId > 1); + const oneNodeAuditedBelow = await scan( + [auditedNode1.nodeInternalId, auditedNode4.nodeInternalId], + [ + Number(auditedNode1.auditedThroughBlockInternalId), + blockInternalId - 1, + ] + ); + t.deepEqual( + oneNodeAuditedBelow.incompleteBlocks.map((block) => block.hash), + [historicalRepairBlockHash] + ); + } finally { + await client.query( + /* sql */ ` + INSERT INTO block_transaction + (block_internal_id, transaction_internal_id, transaction_index) + VALUES ($1, $2, $3); + `, + [ + removedRow.blockInternalId, + removedRow.transactionInternalId, + removedRow.transactionIndex, + ] + ); + await freshDbPool.end(); + if (originalPostgresConnectionString === undefined) { + delete process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + } else { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + originalPostgresConnectionString; + } + } + } +); + test.serial( '[e2e] handles empty headers messages (fully-synced)', async (t) => { From 9bcbdb837f4d337b7b7252f399e3e68a47c38b68 Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 21 Oct 2026 09:30:50 +0100 Subject: [PATCH 62/63] perf(agent): optional binary COPY write path (CHAINGRAPH_WRITE_PATH=copy) (#13) Adds an opt-in block write path (CHAINGRAPH_WRITE_PATH=copy, default sql) that streams rows with binary COPY into per-connection temporary staging tables, pipelined in one round trip, then inserts them with the same statements and conflict handling as the sql path, in one transaction per block. CI runs the full suite on both write paths; adds encoder unit tests and a real-server COPY failure-recovery test. Co-Authored-By: Claude Opus 5.5 --- .cspell.json | 3 + .github/workflows/ci.yaml | 9 +- defaults.env | 3 + src/components/block-copy-rows.spec.ts | 192 ++++++++++ src/components/block-copy-rows.ts | 200 ++++++++++ src/components/pg-binary-copy.e2e.spec.ts | 80 ++++ src/components/pg-binary-copy.spec.ts | 244 ++++++++++++ src/components/pg-binary-copy.ts | 434 ++++++++++++++++++++++ src/config.ts | 20 + src/db.ts | 264 +++++++++++-- 10 files changed, 1424 insertions(+), 25 deletions(-) create mode 100644 src/components/block-copy-rows.spec.ts create mode 100644 src/components/block-copy-rows.ts create mode 100644 src/components/pg-binary-copy.e2e.spec.ts create mode 100644 src/components/pg-binary-copy.spec.ts create mode 100644 src/components/pg-binary-copy.ts diff --git a/.cspell.json b/.cspell.json index 6a7c1519..ba35d020 100644 --- a/.cspell.json +++ b/.cspell.json @@ -71,6 +71,7 @@ "peerready", "peersendheaders", "pgadmin", + "PGCOPY", "pghero", "pino", "pkey", @@ -97,10 +98,12 @@ "sendheaders", "statefulset", "statefulsets", + "subquery", "symlinks", "templating", "txindex", "unintuitive", + "unnest", "unpruned", "upserting", "UTXO", diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3601f263..95ef46d1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,6 +4,13 @@ on: [push, pull_request] jobs: e2e-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + write-path: [sql, copy] + name: e2e-tests (write path ${{ matrix.write-path }}) + env: + CHAINGRAPH_WRITE_PATH: ${{ matrix.write-path }} services: postgres: image: postgres:14 @@ -38,7 +45,7 @@ jobs: uses: actions/upload-artifact@v7 if: always() with: - name: e2e-test-log + name: e2e-test-log-${{ matrix.write-path }} path: data/chaingraph/log-e2e.ndjson check-yarn-cache: diff --git a/defaults.env b/defaults.env index 5bbfb237..79925c43 100644 --- a/defaults.env +++ b/defaults.env @@ -12,6 +12,9 @@ CHAINGRAPH_POSTGRES_MAX_CONNECTIONS= # In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default. CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT=true +# How block data is written to Postgres: `sql` sends each block as multi-row `INSERT … VALUES` statements; `copy` streams the rows with binary `COPY` into per-connection temporary staging tables and inserts from there (same constraints and `ON CONFLICT DO NOTHING` semantics, much less SQL text to build, send, parse and plan). +CHAINGRAPH_WRITE_PATH=sql + # The target size (in MB) of the buffer which holds downloaded blocks waiting to be saved to the database. This primarily affects memory usage during the initial chain sync. # For best performance, this should be around `CHAINGRAPH_POSTGRES_MAX_CONNECTIONS * maximum block size`, while leaving enough memory available to the host machine. If not set, Chaingraph will measure free memory at startup and attempt to select a reasonable value. CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB= diff --git a/src/components/block-copy-rows.spec.ts b/src/components/block-copy-rows.spec.ts new file mode 100644 index 00000000..2ce5a898 --- /dev/null +++ b/src/components/block-copy-rows.spec.ts @@ -0,0 +1,192 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import test from 'ava'; + +import type { + ChaingraphBlock, + ChaingraphTransaction, +} from '../types/chaingraph.js'; + +import { + copyStageTableSql, + createStageTablesSql, + encodeStageBlockTransactions, + encodeStageInputs, + encodeStageOutputs, + encodeStageTransactions, + stageTableColumns, +} from './block-copy-rows.js'; +import { decodeBinaryCopy } from './pg-binary-copy.js'; + +const hash = (byte: string) => byte.repeat(32); +const int8 = (field: Buffer | null | undefined) => field!.readBigInt64BE(0); +const hex = (field: Buffer | null | undefined) => field!.toString('hex'); + +const coinbase: ChaingraphTransaction = { + hash: hash('aa'), + inputs: [ + { + outpointIndex: 0xffffffff, + outpointTransactionHash: hash('00'), + sequenceNumber: 0xffffffff, + unlockingBytecode: '03a0bb0d', + }, + ], + isCoinbase: true, + locktime: 0, + outputs: [{ lockingBytecode: '', valueSatoshis: 625000000n }], + sizeBytes: 100, + version: 1, +}; + +const tokenTransaction: ChaingraphTransaction = { + hash: hash('bb'), + inputs: [ + { + outpointIndex: 1, + outpointTransactionHash: hash('cc'), + sequenceNumber: 0, + unlockingBytecode: '', + }, + { + outpointIndex: 0, + outpointTransactionHash: hash('dd'), + sequenceNumber: 0xfffffffe, + unlockingBytecode: '0051', + }, + ], + isCoinbase: false, + locktime: 500000000, + outputs: [ + { + fungibleTokenAmount: 9223372036854775807n, + lockingBytecode: '76a914', + nonfungibleTokenCapability: 'minting', + nonfungibleTokenCommitment: '', + tokenCategory: hash('ee'), + valueSatoshis: 1000n, + }, + { + lockingBytecode: '6a', + nonfungibleTokenCapability: 'none', + nonfungibleTokenCommitment: '01020304', + tokenCategory: hash('ff'), + valueSatoshis: 0n, + }, + { + fungibleTokenAmount: 1n, + lockingBytecode: 'aa20', + tokenCategory: hash('11'), + valueSatoshis: 800n, + }, + ], + sizeBytes: 300, + version: 2, +}; + +test('encodeStageTransactions: one row per transaction in column order', (t) => { + const rows = decodeBinaryCopy( + encodeStageTransactions([coinbase, tokenTransaction]) + ); + t.is(rows.length, 2); + t.is(rows[0]!.length, stageTableColumns.chaingraph_stage_transaction.length); + t.deepEqual( + rows.map((row) => [ + hex(row[0]), + int8(row[1]), + int8(row[2]), + int8(row[3]), + row[4]![0], + ]), + [ + [hash('aa'), 1n, 0n, 100n, 1], + [hash('bb'), 2n, 500000000n, 300n, 0], + ] + ); +}); + +test('encodeStageInputs: indexes, uint32 sequence numbers and empty bytecode', (t) => { + const rows = decodeBinaryCopy( + encodeStageInputs([coinbase, tokenTransaction]) + ); + t.deepEqual( + rows.map((row) => [ + hex(row[0]), + int8(row[1]), + int8(row[2]), + int8(row[3]), + hex(row[4]), + hex(row[5]), + ]), + [ + [hash('aa'), 0n, 4294967295n, 4294967295n, hash('00'), '03a0bb0d'], + [hash('bb'), 0n, 1n, 0n, hash('cc'), ''], + [hash('bb'), 1n, 0n, 4294967294n, hash('dd'), '0051'], + ] + ); +}); + +test('encodeStageOutputs: token fields, enum labels and NULLs', (t) => { + const rows = decodeBinaryCopy( + encodeStageOutputs([coinbase, tokenTransaction]) + ); + t.is(rows[0]!.length, stageTableColumns.chaingraph_stage_output.length); + t.deepEqual( + rows.map((row) => [ + hex(row[0]), + int8(row[1]), + int8(row[2]), + hex(row[3]), + row[4] === null ? null : hex(row[4]), + row[5] === null ? null : int8(row[5]), + row[6] === null ? null : row[6]!.toString('utf8'), + row[7] === null ? null : hex(row[7]), + ]), + [ + [hash('aa'), 0n, 625000000n, '', null, null, null, null], + [ + hash('bb'), + 0n, + 1000n, + '76a914', + hash('ee'), + 9223372036854775807n, + 'minting', + '', + ], + [hash('bb'), 1n, 0n, '6a', hash('ff'), null, 'none', '01020304'], + [hash('bb'), 2n, 800n, 'aa20', hash('11'), 1n, null, null], + ] + ); +}); + +test('encodeStage*: empty transaction lists encode zero rows', (t) => { + t.deepEqual(decodeBinaryCopy(encodeStageTransactions([])), []); + t.deepEqual(decodeBinaryCopy(encodeStageInputs([])), []); + t.deepEqual(decodeBinaryCopy(encodeStageOutputs([])), []); +}); + +test('encodeStageBlockTransactions: every transaction with its block index', (t) => { + const block = { + transactions: [coinbase, tokenTransaction], + } as unknown as ChaingraphBlock; + const rows = decodeBinaryCopy(encodeStageBlockTransactions(block)); + t.deepEqual( + rows.map((row) => [hex(row[0]), int8(row[1])]), + [ + [hash('aa'), 0n], + [hash('bb'), 1n], + ] + ); +}); + +test('stage table SQL', (t) => { + t.is( + copyStageTableSql('chaingraph_stage_block_transaction'), + 'COPY pg_temp.chaingraph_stage_block_transaction (hash, transaction_index) FROM STDIN (FORMAT binary)' + ); + t.regex( + createStageTablesSql, + /CREATE TEMP TABLE IF NOT EXISTS chaingraph_stage_output \(transaction_hash bytea, .*nonfungible_token_capability enum_nonfungible_token_capability, nonfungible_token_commitment bytea\) ON COMMIT DELETE ROWS;/u + ); + t.is(createStageTablesSql.split('\n').length, 4); +}); diff --git a/src/components/block-copy-rows.ts b/src/components/block-copy-rows.ts new file mode 100644 index 00000000..02897df8 --- /dev/null +++ b/src/components/block-copy-rows.ts @@ -0,0 +1,200 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +/** + * Binary COPY encoders for the per-connection staging tables used by the + * `copy` write path (`CHAINGRAPH_WRITE_PATH=copy`, see `saveBlock` in + * `db.ts`). Column order here must match `stageTableColumns`. + */ +import type { + ChaingraphBlock, + ChaingraphTransaction, +} from '../types/chaingraph.js'; + +import { BinaryCopyWriter } from './pg-binary-copy.js'; + +/** + * Temporary staging tables, created once per pooled connection. Rows are + * removed at every COMMIT (and discarded by ROLLBACK), so each block's DB + * transaction starts with empty tables. + * + * Names are prefixed to avoid shadowing permanent tables (`pg_temp` is + * searched first) and always referenced as `pg_temp.`. + */ +export const stageTableColumns = { + /* eslint-disable camelcase, @typescript-eslint/naming-convention */ + chaingraph_stage_block_transaction: [ + ['hash', 'bytea'], + ['transaction_index', 'bigint'], + ], + chaingraph_stage_input: [ + ['transaction_hash', 'bytea'], + ['input_index', 'bigint'], + ['outpoint_index', 'bigint'], + ['sequence_number', 'bigint'], + ['outpoint_transaction_hash', 'bytea'], + ['unlocking_bytecode', 'bytea'], + ], + chaingraph_stage_output: [ + ['transaction_hash', 'bytea'], + ['output_index', 'bigint'], + ['value_satoshis', 'bigint'], + ['locking_bytecode', 'bytea'], + ['token_category', 'bytea'], + ['fungible_token_amount', 'bigint'], + ['nonfungible_token_capability', 'enum_nonfungible_token_capability'], + ['nonfungible_token_commitment', 'bytea'], + ], + chaingraph_stage_transaction: [ + ['hash', 'bytea'], + ['version', 'bigint'], + ['locktime', 'bigint'], + ['size_bytes', 'bigint'], + ['is_coinbase', 'boolean'], + ], + /* eslint-enable camelcase, @typescript-eslint/naming-convention */ +} as const; + +export type StageTable = keyof typeof stageTableColumns; + +export const createStageTablesSql = Object.entries(stageTableColumns) + .map( + ([table, columns]) => + `CREATE TEMP TABLE IF NOT EXISTS ${table} (${columns + .map(([name, type]) => `${name} ${type}`) + .join(', ')}) ON COMMIT DELETE ROWS;` + ) + .join('\n'); + +const buildCopyStageTableSql = (table: StageTable) => + `COPY pg_temp.${table} (${stageTableColumns[table] + .map(([name]) => name) + .join(', ')}) FROM STDIN (FORMAT binary)`; + +const copyStageTableStatements = Object.fromEntries( + Object.keys(stageTableColumns).map((table) => [ + table, + buildCopyStageTableSql(table as StageTable), + ]) +) as { [table in StageTable]: string }; + +export const copyStageTableSql = (table: StageTable) => + copyStageTableStatements[table]; + +const hashBytes = 32; +const copyHeaderAndTrailerBytes = 64; +const transactionRowBytes = 2 + 5 * 4 + hashBytes + 3 * 8 + 1; +const blockTransactionRowBytes = 2 + 2 * 4 + hashBytes + 8; +const hexCharsPerByte = 2; +const inputRowFixedBytes = 2 + 6 * 4 + 2 * hashBytes + 3 * 8; +const outputRowFixedBytes = 2 + 8 * 4 + hashBytes + 2 * 8; + +export const encodeStageTransactions = ( + transactions: ChaingraphTransaction[] +) => { + const writer = new BinaryCopyWriter( + transactions.length * transactionRowBytes + copyHeaderAndTrailerBytes + ); + transactions.forEach((transaction) => { + writer + .startRow(stageTableColumns.chaingraph_stage_transaction.length) + .hexBytea(transaction.hash) + .int8(transaction.version) + .int8(transaction.locktime) + .int8(transaction.sizeBytes) + .boolean(transaction.isCoinbase); + }); + return writer.finish(); +}; + +export const encodeStageInputs = (transactions: ChaingraphTransaction[]) => { + const expectedBytes = transactions.reduce( + (total, transaction) => + total + + transaction.inputs.reduce( + (sum, input) => + sum + + inputRowFixedBytes + + input.unlockingBytecode.length / hexCharsPerByte, + 0 + ), + copyHeaderAndTrailerBytes + ); + const writer = new BinaryCopyWriter(expectedBytes); + transactions.forEach((transaction) => { + transaction.inputs.forEach((input, inputIndex) => { + writer + .startRow(stageTableColumns.chaingraph_stage_input.length) + .hexBytea(transaction.hash) + .int8(inputIndex) + .int8(input.outpointIndex) + .int8(input.sequenceNumber) + .hexBytea(input.outpointTransactionHash) + .hexBytea(input.unlockingBytecode); + }); + }); + return writer.finish(); +}; + +export const encodeStageOutputs = (transactions: ChaingraphTransaction[]) => { + const expectedBytes = transactions.reduce( + (total, transaction) => + total + + transaction.outputs.reduce( + (sum, output) => + sum + + outputRowFixedBytes + + output.lockingBytecode.length / hexCharsPerByte, + 0 + ), + copyHeaderAndTrailerBytes + ); + const writer = new BinaryCopyWriter(expectedBytes); + transactions.forEach((transaction) => { + transaction.outputs.forEach((output, outputIndex) => { + writer + .startRow(stageTableColumns.chaingraph_stage_output.length) + .hexBytea(transaction.hash) + .int8(outputIndex) + .int8(output.valueSatoshis) + .hexBytea(output.lockingBytecode); + if (output.tokenCategory === undefined) { + writer.null(); + } else { + writer.hexBytea(output.tokenCategory); + } + if (output.fungibleTokenAmount === undefined) { + writer.null(); + } else { + writer.int8(output.fungibleTokenAmount); + } + if (output.nonfungibleTokenCapability === undefined) { + writer.null(); + } else { + writer.text(output.nonfungibleTokenCapability); + } + if (output.nonfungibleTokenCommitment === undefined) { + writer.null(); + } else { + writer.hexBytea(output.nonfungibleTokenCommitment); + } + }); + }); + return writer.finish(); +}; + +/** + * Every transaction in the block (including those already saved), with its + * index in the block. + */ +export const encodeStageBlockTransactions = (block: ChaingraphBlock) => { + const writer = new BinaryCopyWriter( + block.transactions.length * blockTransactionRowBytes + + copyHeaderAndTrailerBytes + ); + block.transactions.forEach((transaction, transactionIndex) => { + writer + .startRow(stageTableColumns.chaingraph_stage_block_transaction.length) + .hexBytea(transaction.hash) + .int8(transactionIndex); + }); + return writer.finish(); +}; diff --git a/src/components/pg-binary-copy.e2e.spec.ts b/src/components/pg-binary-copy.e2e.spec.ts new file mode 100644 index 00000000..0c5b2727 --- /dev/null +++ b/src/components/pg-binary-copy.e2e.spec.ts @@ -0,0 +1,80 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import type { TestFn } from 'ava'; +import baseTest from 'ava'; +import pg from 'pg'; + +import { BinaryCopyWriter, copyFromBuffers } from './pg-binary-copy.js'; + +const test = baseTest as TestFn<{ client: pg.Client }>; +const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; +const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; + +/* + * Runs against a real server, using only temporary tables (no persistent + * tables are created or modified). + */ +test.before(async (t) => { + t.context.client = new pg.Client({ + connectionString: `postgres://chaingraph:very_insecure_postgres_password@${host}:${port}/postgres`, + }); + await t.context.client.connect(); + await t.context.client.query(/* sql */ ` +SET statement_timeout = '5s'; +CREATE TEMP TABLE stage_a (id bigint NOT NULL); +CREATE TEMP TABLE stage_b (id bigint NOT NULL); +`); +}); + +test.after.always(async (t) => { + await t.context.client.end(); +}); + +const rows = (values: (number | null)[]) => + values + .reduce( + (writer, value) => + value === null + ? writer.startRow(1).null() + : writer.startRow(1).int8(value), + new BinaryCopyWriter() + ) + .finish(); +const copyA = 'COPY pg_temp.stage_a (id) FROM STDIN WITH (FORMAT binary)'; +const copyB = 'COPY pg_temp.stage_b (id) FROM STDIN WITH (FORMAT binary)'; +const ids = async (client: pg.Client, table: string) => + ( + await client.query<{ id: string }>( + `SELECT id FROM pg_temp.${table} ORDER BY id;` + ) + ).rows.map((row) => Number(row.id)); + +test.serial( + '[e2e] copyFromBuffers: a failed later COPY rolls back and the connection stays usable', + async (t) => { + const { client } = t.context; + await client.query('BEGIN;'); + const error = await t.throwsAsync( + copyFromBuffers(client, [ + { messages: rows([1, 2]), statement: copyA }, + { messages: rows([3, null]), statement: copyB }, + { messages: rows([5]), statement: copyA }, + ]) + ); + t.regex(error?.message ?? '', /null value/u); + await client.query('ROLLBACK;'); + t.deepEqual(await ids(client, 'stage_a'), []); + t.deepEqual(await ids(client, 'stage_b'), []); + + await client.query('BEGIN;'); + t.deepEqual( + await copyFromBuffers(client, [ + { messages: rows([10, 11]), statement: copyA }, + { messages: rows([12]), statement: copyB }, + ]), + [2, 1] + ); + await client.query('COMMIT;'); + t.deepEqual(await ids(client, 'stage_a'), [10, 11]); + t.deepEqual(await ids(client, 'stage_b'), [12]); + } +); diff --git a/src/components/pg-binary-copy.spec.ts b/src/components/pg-binary-copy.spec.ts new file mode 100644 index 00000000..3110f2e2 --- /dev/null +++ b/src/components/pg-binary-copy.spec.ts @@ -0,0 +1,244 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import test from 'ava'; + +import { + binaryCopySignature, + BinaryCopyWriter, + copyDataPayload, + CopyFromBuffersQuery, + decodeBinaryCopy, +} from './pg-binary-copy.js'; + +const header = Buffer.concat([ + binaryCopySignature, + Buffer.from('0000000000000000', 'hex'), +]); +const trailer = Buffer.from('ffff', 'hex'); + +const copyDone = Buffer.from('6300000004', 'hex'); +const copyData = (payload: Buffer) => { + const messageHeader = Buffer.alloc(5); + messageHeader[0] = 0x64; + messageHeader.writeInt32BE(4 + payload.length, 1); + return Buffer.concat([messageHeader, payload, copyDone]); +}; + +test('BinaryCopyWriter: empty payload is header + trailer, framed as CopyData + CopyDone', (t) => { + const messages = new BinaryCopyWriter().finish(); + t.deepEqual(messages, copyData(Buffer.concat([header, trailer]))); + t.deepEqual(copyDataPayload(messages), Buffer.concat([header, trailer])); + t.deepEqual(decodeBinaryCopy(messages), []); +}); + +test('copyDataPayload: rejects anything but one CopyData + CopyDone', (t) => { + const messages = new BinaryCopyWriter().finish(); + t.throws(() => copyDataPayload(messages.subarray(0, messages.length - 1))); + t.throws(() => copyDataPayload(Buffer.concat([messages, copyDone]))); + const wrongCode = Buffer.from(messages); + wrongCode[0] = 0x51; + t.throws(() => copyDataPayload(wrongCode)); +}); + +test('BinaryCopyWriter: exact wire bytes for every field type', (t) => { + const payload = new BinaryCopyWriter() + .startRow(6) + .hexBytea('c0de') + .int8(1) + .int8(-2n) + .boolean(true) + .text('minting') + .null() + .finish(); + t.deepEqual( + copyDataPayload(payload), + Buffer.concat([ + header, + Buffer.from('0006', 'hex'), + Buffer.from('00000002c0de', 'hex'), + Buffer.from('000000080000000000000001', 'hex'), + Buffer.from('00000008fffffffffffffffe', 'hex'), + Buffer.from('0000000101', 'hex'), + Buffer.from('000000076d696e74696e67', 'hex'), + Buffer.from('ffffffff', 'hex'), + trailer, + ]) + ); +}); + +const int8Cases: [bigint | number, string][] = [ + [0, '0000000000000000'], + [1, '0000000000000001'], + [-1, 'ffffffffffffffff'], + [0xffffffff, '00000000ffffffff'], + [0x1_0000_0000, '0000000100000000'], + [-0x1_0000_0000, 'ffffffff00000000'], + // cspell:disable-next-line + [-0x1_0000_0001, 'fffffffeffffffff'], + [Number.MAX_SAFE_INTEGER, '001fffffffffffff'], + [Number.MIN_SAFE_INTEGER, 'ffe0000000000001'], + [9223372036854775807n, '7fffffffffffffff'], + [-9223372036854775808n, '8000000000000000'], + [2_100_000_000_000_000n, '000775f05a074000'], +]; + +test('BinaryCopyWriter: int8 round-trips (number and bigint)', (t) => { + const writer = new BinaryCopyWriter(1); + int8Cases.forEach(([value]) => writer.startRow(1).int8(value)); + const rows = decodeBinaryCopy(writer.finish()); + t.is(rows.length, int8Cases.length); + rows.forEach((row, index) => { + const [value, hex] = int8Cases[index]!; + t.is(row[0]!.toString('hex'), hex); + t.is(row[0]!.readBigInt64BE(0), BigInt(value)); + }); +}); + +test('BinaryCopyWriter: int8 rejects unsafe or fractional numbers', (t) => { + t.throws(() => new BinaryCopyWriter().int8(Number.MAX_SAFE_INTEGER + 1)); + t.throws(() => new BinaryCopyWriter().int8(1.5)); + t.throws(() => new BinaryCopyWriter().int8(NaN)); + t.throws(() => new BinaryCopyWriter().int8(2n ** 63n)); +}); + +test('BinaryCopyWriter: bytea round-trips (empty, hex, raw, large) and rejects bad hex', (t) => { + const large = Buffer.alloc(3 * 1024 * 1024, 0xab); + const rows = decodeBinaryCopy( + new BinaryCopyWriter(1) + .startRow(4) + .hexBytea('') + .hexBytea('00ff10') + .bytea(Uint8Array.from([1, 2, 3])) + .hexBytea(large.toString('hex')) + .finish() + ); + t.is(rows.length, 1); + t.deepEqual(rows[0]![0], Buffer.alloc(0)); + t.deepEqual(rows[0]![1], Buffer.from('00ff10', 'hex')); + t.deepEqual(rows[0]![2], Buffer.from([1, 2, 3])); + t.true(rows[0]![3]!.equals(large)); + t.throws(() => new BinaryCopyWriter().hexBytea('abc')); + t.throws(() => new BinaryCopyWriter().hexBytea('zz')); +}); + +test('BinaryCopyWriter: booleans, enum labels and nulls', (t) => { + const rows = decodeBinaryCopy( + new BinaryCopyWriter() + .startRow(5) + .boolean(false) + .boolean(true) + .text('none') + .text('mutable') + .null() + .finish() + ); + t.deepEqual(rows, [ + [ + Buffer.from([0]), + Buffer.from([1]), + Buffer.from('none'), + Buffer.from('mutable'), + null, + ], + ]); +}); + +test('BinaryCopyWriter: grows past its initial capacity', (t) => { + const writer = new BinaryCopyWriter(1); + const rowCount = 10_000; + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let index = 0; index < rowCount; index += 1) { + writer.startRow(2).int8(index).hexBytea('0102'); + } + const rows = decodeBinaryCopy(writer.finish()); + t.is(writer.rowCount, rowCount); + t.is(rows.length, rowCount); + t.is(rows[rowCount - 1]![0]!.readBigInt64BE(0), BigInt(rowCount - 1)); +}); + +const parseFrontendMessages = (bytes: Buffer) => { + const messages: { code: string; body: Buffer }[] = []; + // eslint-disable-next-line functional/no-let + let offset = 0; + // eslint-disable-next-line functional/no-loop-statement + while (offset < bytes.length) { + const code = String.fromCharCode(bytes[offset]!); + const length = bytes.readInt32BE(offset + 1); + messages.push({ + body: bytes.subarray(offset + 5, offset + 1 + length), + code, + }); + offset += 1 + length; + } + return messages; +}; + +const mockConnection = () => { + const writes: Buffer[] = []; + const events: string[] = []; + const connection = { + query: (text: string) => { + const body = Buffer.from(`${text}\0`); + const queryHeader = Buffer.alloc(5); + queryHeader[0] = 'Q'.charCodeAt(0); + queryHeader.writeInt32BE(4 + body.length, 1); + writes.push(queryHeader, body); + }, + stream: { + cork: () => events.push('cork'), + uncork: () => events.push('uncork'), + write: (chunk: Buffer) => { + writes.push(Buffer.from(chunk)); + return true; + }, + }, + }; + return { connection, events, writes }; +}; + +test('CopyFromBuffersQuery: one corked write of Query + CopyData/CopyDone per statement', (t) => { + const first = new BinaryCopyWriter().startRow(1).int8(7).finish(); + const second = new BinaryCopyWriter().finish(); + const { connection, events, writes } = mockConnection(); + const query = new CopyFromBuffersQuery([ + { messages: first, statement: 'COPY a FROM STDIN' }, + { messages: second, statement: 'COPY b FROM STDIN' }, + ]); + const results: [Error | undefined, (number | null)[]][] = []; + query.callback = (err, rowCounts) => results.push([err, rowCounts]); + query.submit(connection as never); + t.deepEqual(events, ['cork', 'uncork']); + const messages = parseFrontendMessages(Buffer.concat(writes)); + t.deepEqual( + messages.map((message) => message.code), + ['Q', 'd', 'c', 'd', 'c'] + ); + t.is( + messages[0]!.body.toString(), + 'COPY a FROM STDIN;\nCOPY b FROM STDIN;\0' + ); + t.deepEqual(messages[1]!.body, copyDataPayload(first)); + t.deepEqual(messages[3]!.body, copyDataPayload(second)); + t.deepEqual(messages[2]!.body.length, 0); + query.handleCopyInResponse(); + query.handleCommandComplete({ text: 'COPY 1' }); + query.handleCopyInResponse(); + query.handleCommandComplete({ text: 'COPY 0' }); + query.handleReadyForQuery(); + query.handleReadyForQuery(); + t.deepEqual(results, [[undefined, [1, 0]]]); +}); + +test('CopyFromBuffersQuery: errors reach the callback once', (t) => { + const query = new CopyFromBuffersQuery([ + { + messages: new BinaryCopyWriter().finish(), + statement: 'COPY x FROM STDIN', + }, + ]); + const results: (Error | undefined)[] = []; + query.callback = (err) => results.push(err); + const error = new Error('boom'); + query.handleError(error); + query.handleReadyForQuery(); + t.deepEqual(results, [error]); +}); diff --git a/src/components/pg-binary-copy.ts b/src/components/pg-binary-copy.ts new file mode 100644 index 00000000..3a1a6770 --- /dev/null +++ b/src/components/pg-binary-copy.ts @@ -0,0 +1,434 @@ +/* eslint-disable max-classes-per-file */ +/** + * Minimal support for Postgres `COPY … FROM STDIN (FORMAT binary)` on top of + * the `pg` driver, without additional dependencies. + * + * - `BinaryCopyWriter` encodes rows in the binary COPY file format + * (https://www.postgresql.org/docs/current/sql-copy.html#id-1.9.3.55.9.4) + * directly inside the frontend protocol messages which carry them (one + * CopyData message followed by CopyDone), so they can be written to the + * socket without another copy. + * - `CopyFromBuffersQuery` is a `pg` "submittable" which runs one or more + * `COPY … FROM STDIN` statements and pipelines their encoded data (the same + * protocol hooks `pg-copy-streams` uses). + */ +import type pg from 'pg'; + +/** + * `PGCOPY\n\377\r\n\0`, followed by a 32-bit flags field (0) and a 32-bit + * header extension length (0). + */ +export const binaryCopySignature = Buffer.from([ + /* eslint-disable @typescript-eslint/no-magic-numbers */ + 0x50, 0x47, 0x43, 0x4f, 0x50, 0x59, 0x0a, 0xff, 0x0d, 0x0a, 0x00, + /* eslint-enable @typescript-eslint/no-magic-numbers */ +]); +const flagsAndExtensionBytes = 8; +const headerBytes = binaryCopySignature.length + flagsAndExtensionBytes; +/** A frontend message's type byte plus its int32 length. */ +const messageHeaderBytes = 5; +const messageLengthBytes = 4; +/** CopyData header before the data, CopyDone after it. */ +const framingBytes = messageHeaderBytes + messageHeaderBytes; +/** `d` */ +const copyDataCode = 0x64; +/** `c` */ +const copyDoneCode = 0x63; +/** + * Postgres rejects messages over 1 GB (`PQ_LARGE_MESSAGE_LIMIT`); even a + * 32 MB block encodes to well under 100 MB per table. + */ +const maxCopyDataBytes = 0x3fff_fff0; +const fieldCountBytes = 2; +const fieldLengthBytes = 4; +const int8Bytes = 8; +const booleanBytes = 1; +const nullFieldLength = -1; +const trailer = -1; +const uint32Range = 0x1_0000_0000; +const hexCharsPerByte = 2; +const maxErrorHexChars = 64; +const bytesPerKilobyte = 1024; +const initialCapacityKilobytes = 64; +const initialCapacity = initialCapacityKilobytes * bytesPerKilobyte; +const growthFactor = 2; + +/** + * Encode rows in the Postgres binary COPY format. Fields must be written in + * the column order of the `COPY` statement. Integer types are written as + * `int8`, so every integer column must be `bigint`; `enum` values are sent as + * their label (that is the binary format of `enum_send`/`enum_recv`). + * + * The data is written after a reserved CopyData message header, and `finish` + * appends a CopyDone message: the result is ready to send as-is after a + * `COPY … FROM STDIN` Query message. + */ +export class BinaryCopyWriter { + buffer: Buffer; + + offset: number; + + rowCount = 0; + + constructor(expectedSizeBytes = initialCapacity) { + this.buffer = Buffer.allocUnsafe( + Math.max( + expectedSizeBytes + framingBytes, + framingBytes + headerBytes + fieldCountBytes + ) + ); + const start = messageHeaderBytes; + binaryCopySignature.copy(this.buffer, start); + this.buffer.writeInt32BE(0, start + binaryCopySignature.length); + this.buffer.writeInt32BE( + 0, + start + binaryCopySignature.length + fieldLengthBytes + ); + this.offset = start + headerBytes; + } + + ensure(additionalBytes: number) { + const required = this.offset + additionalBytes; + if (required <= this.buffer.length) { + return; + } + const grown = Buffer.allocUnsafe( + Math.max(required, this.buffer.length * growthFactor) + ); + this.buffer.copy(grown, 0, 0, this.offset); + this.buffer = grown; + } + + startRow(fieldCount: number) { + this.ensure(fieldCountBytes); + this.buffer.writeInt16BE(fieldCount, this.offset); + this.offset += fieldCountBytes; + this.rowCount += 1; + return this; + } + + null() { + this.ensure(fieldLengthBytes); + this.buffer.writeInt32BE(nullFieldLength, this.offset); + this.offset += fieldLengthBytes; + return this; + } + + /** + * Write a `bigint` (int8) field. Numbers must be safe integers. + */ + int8(value: bigint | number) { + this.ensure(fieldLengthBytes + int8Bytes); + this.buffer.writeInt32BE(int8Bytes, this.offset); + this.offset += fieldLengthBytes; + if (typeof value === 'bigint') { + this.buffer.writeBigInt64BE(value, this.offset); + } else { + if (!Number.isSafeInteger(value)) { + // eslint-disable-next-line functional/no-throw-statement + throw new RangeError(`Cannot encode ${value} as int8.`); + } + const high = Math.floor(value / uint32Range); + const low = value - high * uint32Range; + this.buffer.writeInt32BE(high, this.offset); + this.buffer.writeUInt32BE(low, this.offset + fieldLengthBytes); + } + this.offset += int8Bytes; + return this; + } + + boolean(value: boolean) { + this.ensure(fieldLengthBytes + booleanBytes); + this.buffer.writeInt32BE(booleanBytes, this.offset); + this.offset += fieldLengthBytes; + this.buffer[this.offset] = value ? 1 : 0; + this.offset += booleanBytes; + return this; + } + + /** + * Write a `bytea` field from a hex string (without `\x` prefix). + */ + hexBytea(hex: string) { + if (hex.length % hexCharsPerByte !== 0) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Invalid hex (odd length): ${hex.slice(0, maxErrorHexChars)}` + ); + } + const byteLength = hex.length / hexCharsPerByte; + this.ensure(fieldLengthBytes + byteLength); + this.buffer.writeInt32BE(byteLength, this.offset); + this.offset += fieldLengthBytes; + const written = this.buffer.write(hex, this.offset, byteLength, 'hex'); + if (written !== byteLength) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error(`Invalid hex: ${hex.slice(0, maxErrorHexChars)}`); + } + this.offset += byteLength; + return this; + } + + bytea(bytes: Uint8Array) { + this.ensure(fieldLengthBytes + bytes.length); + this.buffer.writeInt32BE(bytes.length, this.offset); + this.offset += fieldLengthBytes; + this.buffer.set(bytes, this.offset); + this.offset += bytes.length; + return this; + } + + /** + * Write a `text` (or `enum` label) field. + */ + text(value: string) { + const byteLength = Buffer.byteLength(value, 'utf8'); + this.ensure(fieldLengthBytes + byteLength); + this.buffer.writeInt32BE(byteLength, this.offset); + this.offset += fieldLengthBytes; + this.buffer.write(value, this.offset, byteLength, 'utf8'); + this.offset += byteLength; + return this; + } + + /** + * Append the trailer and return the frontend messages carrying the data: a + * CopyData message with the complete binary COPY payload, then CopyDone (a + * view of the internal buffer). The writer must not be used afterwards. + */ + finish() { + this.ensure(fieldCountBytes + messageHeaderBytes); + this.buffer.writeInt16BE(trailer, this.offset); + this.offset += fieldCountBytes; + const payloadBytes = this.offset - messageHeaderBytes; + if (payloadBytes > maxCopyDataBytes) { + // eslint-disable-next-line functional/no-throw-statement + throw new RangeError( + `COPY payload of ${payloadBytes} bytes exceeds the CopyData message limit.` + ); + } + this.buffer[0] = copyDataCode; + this.buffer.writeInt32BE(messageLengthBytes + payloadBytes, 1); + this.buffer[this.offset] = copyDoneCode; + this.buffer.writeInt32BE(messageLengthBytes, this.offset + 1); + this.offset += messageHeaderBytes; + return this.buffer.subarray(0, this.offset); + } +} + +const decodeRow = (payload: Buffer, start: number, fieldCount: number) => { + const row: (Buffer | null)[] = []; + // eslint-disable-next-line functional/no-let + let offset = start; + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let field = 0; field < fieldCount; field += 1) { + const length = payload.readInt32BE(offset); + offset += fieldLengthBytes; + const isNull = length === nullFieldLength; + row.push(isNull ? null : payload.subarray(offset, offset + length)); + offset += isNull ? 0 : length; + } + return { offset, row }; +}; + +/** + * Extract the binary COPY payload from the messages built by + * `BinaryCopyWriter.finish` (one CopyData message, then CopyDone). + */ +export const copyDataPayload = (messages: Buffer) => { + const copyDataLength = messages.readInt32BE(1); + const copyDoneOffset = 1 + copyDataLength; + if ( + messages[0] !== copyDataCode || + messages[copyDoneOffset] !== copyDoneCode || + messages.readInt32BE(copyDoneOffset + 1) !== messageLengthBytes || + copyDoneOffset + messageHeaderBytes !== messages.length + ) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error('Expected exactly one CopyData message and CopyDone.'); + } + return messages.subarray(messageHeaderBytes, copyDoneOffset); +}; + +/** + * Decode the messages built by `BinaryCopyWriter.finish`. Only used to verify + * encoders: every non-null field is returned as raw bytes. + */ +export const decodeBinaryCopy = (messages: Buffer) => { + const payload = copyDataPayload(messages); + if ( + !payload.subarray(0, binaryCopySignature.length).equals(binaryCopySignature) + ) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error('Missing binary COPY signature.'); + } + const extensionLength = payload.readInt32BE( + binaryCopySignature.length + fieldLengthBytes + ); + const rows: (Buffer | null)[][] = []; + // eslint-disable-next-line functional/no-let + let offset = headerBytes + extensionLength; + // eslint-disable-next-line functional/no-let + let fieldCount = payload.readInt16BE(offset); + // eslint-disable-next-line functional/no-loop-statement + while (fieldCount !== trailer) { + const { offset: rowEnd, row } = decodeRow( + payload, + offset + fieldCountBytes, + fieldCount + ); + rows.push(row); + offset = rowEnd; + fieldCount = payload.readInt16BE(offset); + } + if (offset + fieldCountBytes !== payload.length) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error('Trailing bytes after binary COPY trailer.'); + } + return rows; +}; + +// eslint-disable-next-line functional/no-mixed-type +interface CopyConnection { + query: (text: string) => void; + stream: { + cork: () => void; + uncork: () => void; + write: (chunk: Buffer) => boolean; + }; +} + +export interface CopyFromStdin { + /** + * A `COPY … FROM STDIN` statement (without trailing semicolon). + */ + statement: string; + /** + * The CopyData and CopyDone messages for this statement, from + * `BinaryCopyWriter.finish`. + */ + messages: Buffer; +} + +/** + * A `pg` submittable which runs one or more `COPY … FROM STDIN` statements in + * a single simple-protocol Query message and pipelines all of their data + * right behind it (Query, then CopyData + CopyDone per statement, in one + * corked socket write). This costs one round trip regardless of the number of + * statements, instead of one round trip per statement plus one per + * CopyInResponse. + * + * Pipelining is safe: if a statement fails, Postgres skips the rest of the + * Query message, sends ReadyForQuery and drops the remaining CopyData and + * CopyDone messages ("Copy-In Mode" in the protocol documentation). + * + * Use via `copyFromBuffers`. + */ +export class CopyFromBuffersQuery implements pg.Submittable { + copies: CopyFromStdin[]; + + rowCounts: (number | null)[] = []; + + error: Error | undefined; + + done = false; + + callback: + | ((err: Error | undefined, rowCounts: (number | null)[]) => void) + | undefined; + + constructor(copies: CopyFromStdin[]) { + this.copies = copies; + } + + finish(err: Error | undefined) { + if (this.done) { + return; + } + this.done = true; + this.callback?.(err, this.rowCounts); + } + + submit(connection: pg.Connection) { + const copyConnection = connection as unknown as CopyConnection; + const { stream } = copyConnection; + stream.cork(); + copyConnection.query( + this.copies.map((copy) => `${copy.statement};`).join('\n') + ); + this.copies.forEach(({ messages }) => { + stream.write(messages); + }); + stream.uncork(); + } + + /** + * The data was already sent in `submit`. + */ + // eslint-disable-next-line class-methods-use-this + handleCopyInResponse() { + /* already sent */ + } + + handleCommandComplete(message: { text?: string }) { + const match = /^COPY (?\d+)$/u.exec(message.text ?? ''); + this.rowCounts.push( + match?.groups?.count === undefined ? null : Number(match.groups.count) + ); + } + + handleReadyForQuery() { + this.finish(this.error); + } + + handleError(err: Error) { + this.error = err; + this.finish(err); + } + + /* The remaining hooks are required by `pg` but unused by COPY FROM. */ + // eslint-disable-next-line class-methods-use-this + handleRowDescription() { + /* unused */ + } + + // eslint-disable-next-line class-methods-use-this + handleDataRow() { + /* unused */ + } + + // eslint-disable-next-line class-methods-use-this + handleEmptyQuery() { + /* unused */ + } + + // eslint-disable-next-line class-methods-use-this + handlePortalSuspended() { + /* unused */ + } + + // eslint-disable-next-line class-methods-use-this + handleCopyData() { + /* unused */ + } +} + +/** + * Run `COPY … FROM STDIN` statements on `client` with already-encoded data, + * in one round trip. Resolves to the row count reported for each statement. + */ +export const copyFromBuffers = async ( + client: pg.ClientBase, + copies: CopyFromStdin[] +) => + new Promise<(number | null)[]>((resolve, reject) => { + const query = new CopyFromBuffersQuery(copies); + query.callback = (err, rowCounts) => { + if (err === undefined) { + resolve(rowCounts); + } else { + reject(err); + } + }; + client.query(query); + }); diff --git a/src/config.ts b/src/config.ts index ff2446b2..e19eff5d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -51,6 +51,7 @@ const expectedOptions = [ 'CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT', 'CHAINGRAPH_TRUSTED_NODES', 'CHAINGRAPH_USER_AGENT', + 'CHAINGRAPH_WRITE_PATH', 'NODE_ENV', ] as const; const requireStringValues = ( @@ -439,6 +440,24 @@ const postgresSynchronousCommit = */ const isProduction = configuration.NODE_ENV === 'production'; +const writePaths = ['sql', 'copy'] as const; +const isWritePath = (value: string): value is (typeof writePaths)[number] => + writePaths.includes(value as (typeof writePaths)[number]); +if (!isWritePath(configuration.CHAINGRAPH_WRITE_PATH)) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Invalid value provided in the 'CHAINGRAPH_WRITE_PATH' environment variable: ${ + configuration.CHAINGRAPH_WRITE_PATH + }. Must be one of the following: ${writePaths.join(', ')}` + ); +} +/** + * Set via the `CHAINGRAPH_WRITE_PATH` environment variable: `sql` (multi-row + * `INSERT … VALUES` statements) or `copy` (binary `COPY` into temporary + * staging tables, then `INSERT … SELECT`). + */ +const chaingraphWritePath = configuration.CHAINGRAPH_WRITE_PATH; + export { blockBufferTargetSizeMb, chaingraphInternalApiPort, @@ -447,6 +466,7 @@ export { chaingraphLogLevelStdout, chaingraphLogLevelPath, chaingraphUserAgent, + chaingraphWritePath, genesisBlocks, incompleteBlockRepairBatchSize, mempoolResync, diff --git a/src/db.ts b/src/db.ts index d976c615..be5eb6c7 100644 --- a/src/db.ts +++ b/src/db.ts @@ -2,11 +2,21 @@ import pg from 'pg'; import type { Agent } from './agent.js'; +import { + copyStageTableSql, + createStageTablesSql, + encodeStageBlockTransactions, + encodeStageInputs, + encodeStageOutputs, + encodeStageTransactions, +} from './components/block-copy-rows.js'; import { computeIndexCreationProgress, indexDefinitions, } from './components/db-utils.js'; +import { copyFromBuffers } from './components/pg-binary-copy.js'; import { + chaingraphWritePath, postgresConnectionString, postgresMaxConnections, postgresSynchronousCommit, @@ -1196,6 +1206,224 @@ export const recordNodeValidation = async ( } }; +/** + * Inside a block's DB transaction: throw (so the caller rolls back) unless + * every transaction of the block was found and linked via block_transaction. + */ +const verifyBlockTransactionsLinked = async ( + client: pg.PoolClient, + block: ChaingraphBlock, + joinedTransactionCount: number +) => { + const linkedBlockTransactionCount = Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*)::bigint AS count + FROM block_transaction + INNER JOIN block ON block.internal_id = block_transaction.block_internal_id + WHERE block.hash = $1; + `, + [Buffer.from(block.hash, 'hex')] + ) + ).rows[0]!.count + ); + if ( + joinedTransactionCount !== block.transactions.length || + linkedBlockTransactionCount !== block.transactions.length + ) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Failed to save all transactions for block ${block.height} (${block.hash}): joined ${joinedTransactionCount}/${block.transactions.length}, linked ${linkedBlockTransactionCount}/${block.transactions.length}.` + ); + } +}; + +/** + * Pooled connections on which the `copy` write path's staging tables exist. + * (A replaced connection is a new client object, so it is set up again.) + */ +const clientsWithStageTables = new WeakSet(); + +/** + * The `copy` write path's statement for transactions, outputs and inputs: the + * same CTE as `addAllTransactions` in `saveBlock`, reading the staged rows + * instead of `VALUES` lists. The text never changes, so Postgres parses it + * cheaply; it is deliberately not a named prepared statement, so each block + * is planned with the current size of the staging tables. + */ +const addAllStagedTransactions = /* sql */ ` +WITH newly_saved_transactions (hash, internal_id) AS ( + INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, version, locktime, size_bytes, is_coinbase FROM pg_temp.chaingraph_stage_transaction + ON CONFLICT ON CONSTRAINT "transaction_hash_key" DO NOTHING + RETURNING hash, internal_id +), +newly_saved_outputs AS ( + INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode, token_category, fungible_token_amount, nonfungible_token_capability, nonfungible_token_commitment) + SELECT transaction_hash, output_index, value_satoshis, locking_bytecode, token_category, fungible_token_amount, nonfungible_token_capability, nonfungible_token_commitment FROM pg_temp.chaingraph_stage_output + WHERE transaction_hash IN (SELECT hash FROM newly_saved_transactions) +), +newly_saved_inputs AS ( + INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode + FROM pg_temp.chaingraph_stage_input val INNER JOIN newly_saved_transactions txs ON val.transaction_hash = txs.hash +) +SELECT COUNT(*) FROM newly_saved_transactions;`; + +/** + * The `copy` write path's block statement: the same CTE as `addBlockQuery` in + * `saveBlock`, with the block's transaction list read from the staging table + * and every other value passed as a parameter (so the text is fixed). + * + * `joined_transactions` is an index lookup per staged transaction (the + * `OFFSET 0` keeps the lateral subquery from being flattened into a join). + * Temporary tables have no statistics, and Postgres assumes at least 10 pages + * for a never-vacuumed table, so a small block's list is estimated at ~1,200 + * rows; with a free choice the planner can pick a hash join over a full scan + * of `transaction`. + */ +const addStagedBlock = /* sql */ ` +WITH accepting_nodes (node_internal_id, accepted_at) AS ( + SELECT * FROM unnest($10::bigint[], $11::timestamp[]) +), +joined_transactions (internal_id, transaction_index) AS ( + SELECT db.internal_id, val.transaction_index + FROM pg_temp.chaingraph_stage_block_transaction val + CROSS JOIN LATERAL ( + SELECT internal_id FROM transaction WHERE transaction.hash = val.hash OFFSET 0 + ) db +), +inserted_block (internal_id) AS ( + INSERT INTO block (height, version, timestamp, hash, previous_block_hash, merkle_root, bits, nonce, size_bytes) + VALUES ($1::bigint, $2::bigint, $3::bigint, $4::bytea, $5::bytea, $6::bytea, $7::bigint, $8::bigint, $9::bigint) + ON CONFLICT ON CONSTRAINT "block_hash_key" DO NOTHING + RETURNING internal_id +), +new_or_existing_block (internal_id) AS ( + SELECT COALESCE ( + (SELECT internal_id FROM inserted_block), + (SELECT internal_id FROM block WHERE block.hash = $4::bytea) + ) +), +inserted_block_transactions AS ( + INSERT INTO block_transaction (block_internal_id, transaction_internal_id, transaction_index) + SELECT blk.internal_id, tx.internal_id, tx.transaction_index + FROM new_or_existing_block blk CROSS JOIN joined_transactions tx + ON CONFLICT ON CONSTRAINT "block_transaction_pkey" DO NOTHING + RETURNING transaction_internal_id +), +inserted_node_blocks AS ( + INSERT INTO node_block (node_internal_id, block_internal_id, accepted_at) + SELECT node.node_internal_id, blk.internal_id, node.accepted_at + FROM new_or_existing_block blk CROSS JOIN accepting_nodes node + ON CONFLICT ON CONSTRAINT "node_block_pkey" DO NOTHING + RETURNING block_internal_id +) +SELECT + (SELECT COUNT(*)::bigint FROM joined_transactions) AS "joinedTransactionCount", + (SELECT COUNT(*)::bigint FROM inserted_block_transactions) AS "insertedBlockTransactionCount", + (SELECT COUNT(*)::bigint FROM inserted_node_blocks) AS "insertedNodeBlockCount";`; + +/** + * The `copy` write path (`CHAINGRAPH_WRITE_PATH=copy`) of `saveBlock`: one DB + * transaction per block, as in the `sql` path. Rows are streamed with binary + * `COPY` into per-connection temporary staging tables, then inserted with the + * same statements, constraints and `ON CONFLICT DO NOTHING` handling. + */ +const saveBlockViaCopy = async ({ + block, + nodeAcceptances, + unknownTransactions, +}: { + block: ChaingraphBlock; + nodeAcceptances: { + nodeInternalId: number; + acceptedAt: Date | null; + }[]; + unknownTransactions: ChaingraphTransaction[]; +}) => { + const stagedTransactions = encodeStageTransactions(unknownTransactions); + const stagedOutputs = encodeStageOutputs(unknownTransactions); + const stagedInputs = encodeStageInputs(unknownTransactions); + const stagedBlockTransactions = encodeStageBlockTransactions(block); + const blockParameters = [ + block.height, + block.version, + block.timestamp, + Buffer.from(block.hash, 'hex'), + Buffer.from(block.previousBlockHash, 'hex'), + Buffer.from(block.merkleRoot, 'hex'), + block.bits, + block.nonce, + block.sizeBytes, + nodeAcceptances.map((acceptance) => acceptance.nodeInternalId), + nodeAcceptances.map((acceptance) => + acceptance.acceptedAt === null + ? null + : acceptance.acceptedAt.toISOString() + ), + ]; + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + if (!clientsWithStageTables.has(client)) { + /* + * Outside of the block's DB transaction, so a rolled-back block can't + * also roll back the table creation. + */ + await client.query(createStageTablesSql); + clientsWithStageTables.add(client); + } + await client.query('BEGIN;'); + await copyFromBuffers(client, [ + { + messages: stagedTransactions, + statement: copyStageTableSql('chaingraph_stage_transaction'), + }, + { + messages: stagedOutputs, + statement: copyStageTableSql('chaingraph_stage_output'), + }, + { + messages: stagedInputs, + statement: copyStageTableSql('chaingraph_stage_input'), + }, + { + messages: stagedBlockTransactions, + statement: copyStageTableSql('chaingraph_stage_block_transaction'), + }, + ]); + const saveTransactionsResult = await client.query<{ count: string }>( + addAllStagedTransactions + ); + const savedTransactionCount = Number(saveTransactionsResult.rows[0]!.count); + const transactionCacheMisses = + unknownTransactions.length - savedTransactionCount; + const addBlockResult = await client.query<{ + insertedBlockTransactionCount: string; + insertedNodeBlockCount: string; + joinedTransactionCount: string; + }>(addStagedBlock, blockParameters); + await verifyBlockTransactionsLinked( + client, + block, + Number(addBlockResult.rows[0]!.joinedTransactionCount) + ); + await client.query('COMMIT;'); + return { + attemptedSavedTransactions: unknownTransactions, + transactionCacheMisses, + }; + } catch (err) { + await client.query('ROLLBACK;'); + // eslint-disable-next-line functional/no-throw-statement + throw err; + } finally { + client.release(); + } +}; + /** * Save a block to the database, inserting all transactions which aren't already * known to exist in the database. (This method should only be used for blocks @@ -1239,6 +1467,14 @@ export const saveBlock = async ({ { alreadySaved: [], unknown: [] } ); + if (chaingraphWritePath === 'copy') { + return saveBlockViaCopy({ + block, + nodeAcceptances, + unknownTransactions: blockTransactions.unknown, + }); + } + const inputs: { inputIndex: number; transactionHash: string; @@ -1426,31 +1662,11 @@ SELECT insertedNodeBlockCount: string; joinedTransactionCount: string; }>(addBlockQuery); - const joinedTransactionCount = Number( - addBlockResult.rows[0]!.joinedTransactionCount + await verifyBlockTransactionsLinked( + client, + block, + Number(addBlockResult.rows[0]!.joinedTransactionCount) ); - const linkedBlockTransactionCount = Number( - ( - await client.query<{ count: string }>( - /* sql */ ` - SELECT COUNT(*)::bigint AS count - FROM block_transaction - INNER JOIN block ON block.internal_id = block_transaction.block_internal_id - WHERE block.hash = $1; - `, - [Buffer.from(block.hash, 'hex')] - ) - ).rows[0]!.count - ); - if ( - joinedTransactionCount !== block.transactions.length || - linkedBlockTransactionCount !== block.transactions.length - ) { - // eslint-disable-next-line functional/no-throw-statement - throw new Error( - `Failed to save all transactions for block ${block.height} (${block.hash}): joined ${joinedTransactionCount}/${block.transactions.length}, linked ${linkedBlockTransactionCount}/${block.transactions.length}.` - ); - } await client.query('COMMIT;'); return { attemptedSavedTransactions, From 0c2ed8612de3aef619d906de1ad68dbbc418b12b Mon Sep 17 00:00:00 2001 From: Richard Brady Date: Wed, 21 Oct 2026 09:56:18 +0100 Subject: [PATCH 63/63] Keep mempool expiry linear when statistics are outdated; fix three flaky tests (#27) The expiry archival statement's descendant walk now looks up node_transaction by its full key behind an OFFSET 0 barrier, so archiving a stale mempool stays linear even when node_transaction statistics are outdated (was quadratic). Adds a plan-based regression test, shrinks the concurrency spec's mempool, and replaces fixed sleeps in two e2e tests with polling for the saved transactions. Co-Authored-By: Claude Opus 5.5 --- src/db.ts | 219 +++++++++++---------- src/e2e/e2e.spec.ts | 74 +++++-- src/e2e/mempool-expiry-concurrency.spec.ts | 118 ++++++++++- 3 files changed, 281 insertions(+), 130 deletions(-) diff --git a/src/db.ts b/src/db.ts index be5eb6c7..fb7080b2 100644 --- a/src/db.ts +++ b/src/db.ts @@ -655,100 +655,20 @@ const acceptedBlockResolution = (node: string, transaction: string) => { }; /** - * Archive expired node_transaction rows, and their same-node descendants, in - * one transaction: take the expiry advisory lock, lock the expired rows, then - * archive them in one statement. Each is a separate command, so (in READ - * COMMITTED) the archival statement's snapshot is taken after both waits: it - * sees block inclusions committed while expiry waited, and no expired row - * deleted meanwhile (e.g. confirmed by block acceptance) can seed the - * descendant walk, as rows it has locked cannot be deleted under it. - * Descendants are only locked by the archival statement itself, after its walk: - * if one was deleted meanwhile (e.g. confirmed, ending the cascade there), the - * walk may have reached its own descendants through it, so the statement - * deletes nothing and the transaction is retried with a fresh snapshot. - * - * An expired row is left in place (not returned) if the transaction, or a - * conflicting spend of one of its outpoints, is already in a block accepted by - * the same node: {@link archiveMempoolTransactionsAcceptedByBlocks} archives - * those with the correct replaced_at (NULL or the block's accepted_at). The - * guard is part of the statement that locks and deletes the row, so it is - * evaluated against the row being archived, not a read taken earlier: a row - * inserted after the agent's accepted-block archive but before its expiry scan - * would otherwise be archived as expired/replaced even though it is confirmed. - * (Such a row is still archived if it descends from another row archived - * here, with the replaced_at described below.) + * The statement archiving a batch of expired node_transaction rows and their + * same-node descendants (see {@link archiveMempoolTransactions}): `$1` to `$4` + * are the batch's node internal IDs, transaction internal IDs, replaced_at + * timestamps and scheduled validated_at timestamps (NULL to skip that check). * - * Descendants are archived here rather than by the history trigger's cascade - * (which then finds none left), for two reasons: - * - a descendant gets the earliest expiry among its archived ancestors, also - * when it is itself in the batch with a later expiry (the trigger only - * reaches descendants still in node_transaction, so it cannot carry an - * ancestor's expiry through a descendant this statement deletes itself); - * - a descendant resolved by a block accepted by the node gets the - * replaced_at the accepted-block archive would give it (NULL if confirmed, - * ending the cascade there; otherwise the block's accepted_at, passed on to - * its own descendants, or NULL if that is unknown, again ending the cascade) - * instead of its ancestor's expiry. - * - * Only expired rows without an expired parent expiring no later seed the - * cascade (the others are reached from that parent with an earlier or equal - * replaced_at), so an expired chain is walked once rather than once per row. - * - * Deadlocks: archiving a mempool transaction also archives its descendants, so - * concurrent single-row expiries of a parent and of its descendants lock the - * same rows in opposite orders (each locks its own row, then the cascade waits - * for the other's) and deadlock; a restarted agent's stale mempool, expiring - * all at once on every pool connection, live-locked this way. Expiry therefore - * runs one batch at a time (see {@link expireMempoolTransaction}), under a - * transaction-level advisory lock (also across agents sharing the database), - * locks the rows it archives in (node, transaction) order (the expired rows, - * then their descendants) before deleting them, and retries a deadlock, - * serialization failure or vanished descendant (see above) against other - * writers (block acceptance, double-spend replacement) with backoff; once - * retries are exhausted it throws, leaving the rows for the next expiration - * scan. + * Each step of the descendant walk looks up the spending transaction's + * node_transaction row by its full key behind an OFFSET 0 planning barrier. + * Without it, Postgres can join the walked rows to node_transaction by node + * alone (then filter by the spending inputs), reading the node's whole mempool + * once per walked row: it does so when node_transaction statistics predate + * the current mempool (e.g. last analyzed just after a block emptied it), so + * archiving a large mempool would take time quadratic in its size. */ -export const archiveMempoolTransactions = async ( - expiries: MempoolTransactionExpiry[] -): Promise => { - if (expiries.length === 0) { - return []; - } - const sorted = [...expiries].sort( - (a, b) => - a.nodeInternalId - b.nodeInternalId || - a.transactionInternalId - b.transactionInternalId - ); - const nodeInternalIds = sorted.map((expiry) => expiry.nodeInternalId); - const transactionInternalIds = sorted.map( - (expiry) => expiry.transactionInternalId - ); - const archiveOnce = async () => { - const client = await pool.connect(); - // eslint-disable-next-line functional/no-try-statement - try { - await client.query('BEGIN;'); - await client.query('SELECT pg_advisory_xact_lock($1::bigint);', [ - expiryAdvisoryLockKey, - ]); - await client.query( - /* sql */ ` -SELECT 1 - FROM node_transaction - JOIN unnest($1::integer[], $2::bigint[]) - AS expiry (node_internal_id, transaction_internal_id) - ON node_transaction.node_internal_id = expiry.node_internal_id - AND node_transaction.transaction_internal_id = expiry.transaction_internal_id - ORDER BY node_transaction.node_internal_id, node_transaction.transaction_internal_id - FOR UPDATE OF node_transaction; -`, - [nodeInternalIds, transactionInternalIds] - ); - const result = await client.query<{ - archived: string | null; - complete: boolean; - }>( - /* sql */ ` +export const mempoolExpiryArchivalStatement = /* sql */ ` WITH RECURSIVE expiry AS ( SELECT node_internal_id, transaction_internal_id, replaced_at, validated_at FROM unnest($1::integer[], $2::bigint[], $3::timestamp[], $4::timestamp[]) @@ -800,9 +720,13 @@ invalidated (node_internal_id, transaction_internal_id, replaced_at) AS ( ON parent_transaction.internal_id = parent.transaction_internal_id JOIN input child_input ON child_input.outpoint_transaction_hash = parent_transaction.hash - JOIN node_transaction child - ON child.node_internal_id = parent.node_internal_id - AND child.transaction_internal_id = child_input.transaction_internal_id + CROSS JOIN LATERAL ( + SELECT node_internal_id, transaction_internal_id + FROM node_transaction + WHERE node_internal_id = parent.node_internal_id + AND transaction_internal_id = child_input.transaction_internal_id + OFFSET 0 + ) child CROSS JOIN LATERAL (${acceptedBlockResolution( 'child.node_internal_id', 'child.transaction_internal_id' @@ -857,14 +781,107 @@ SELECT complete.complete, inserted_history.node_internal_id || ':' || inserted_history.transaction_internal_id AS archived FROM complete LEFT JOIN inserted_history ON true; +`; + +/** + * Archive expired node_transaction rows, and their same-node descendants, in + * one transaction: take the expiry advisory lock, lock the expired rows, then + * archive them in one statement. Each is a separate command, so (in READ + * COMMITTED) the archival statement's snapshot is taken after both waits: it + * sees block inclusions committed while expiry waited, and no expired row + * deleted meanwhile (e.g. confirmed by block acceptance) can seed the + * descendant walk, as rows it has locked cannot be deleted under it. + * Descendants are only locked by the archival statement itself, after its walk: + * if one was deleted meanwhile (e.g. confirmed, ending the cascade there), the + * walk may have reached its own descendants through it, so the statement + * deletes nothing and the transaction is retried with a fresh snapshot. + * + * An expired row is left in place (not returned) if the transaction, or a + * conflicting spend of one of its outpoints, is already in a block accepted by + * the same node: {@link archiveMempoolTransactionsAcceptedByBlocks} archives + * those with the correct replaced_at (NULL or the block's accepted_at). The + * guard is part of the statement that locks and deletes the row, so it is + * evaluated against the row being archived, not a read taken earlier: a row + * inserted after the agent's accepted-block archive but before its expiry scan + * would otherwise be archived as expired/replaced even though it is confirmed. + * (Such a row is still archived if it descends from another row archived + * here, with the replaced_at described below.) + * + * Descendants are archived here rather than by the history trigger's cascade + * (which then finds none left), for two reasons: + * - a descendant gets the earliest expiry among its archived ancestors, also + * when it is itself in the batch with a later expiry (the trigger only + * reaches descendants still in node_transaction, so it cannot carry an + * ancestor's expiry through a descendant this statement deletes itself); + * - a descendant resolved by a block accepted by the node gets the + * replaced_at the accepted-block archive would give it (NULL if confirmed, + * ending the cascade there; otherwise the block's accepted_at, passed on to + * its own descendants, or NULL if that is unknown, again ending the cascade) + * instead of its ancestor's expiry. + * + * Only expired rows without an expired parent expiring no later seed the + * cascade (the others are reached from that parent with an earlier or equal + * replaced_at), so an expired chain is walked once rather than once per row. + * + * Deadlocks: archiving a mempool transaction also archives its descendants, so + * concurrent single-row expiries of a parent and of its descendants lock the + * same rows in opposite orders (each locks its own row, then the cascade waits + * for the other's) and deadlock; a restarted agent's stale mempool, expiring + * all at once on every pool connection, live-locked this way. Expiry therefore + * runs one batch at a time (see {@link expireMempoolTransaction}), under a + * transaction-level advisory lock (also across agents sharing the database), + * locks the rows it archives in (node, transaction) order (the expired rows, + * then their descendants) before deleting them, and retries a deadlock, + * serialization failure or vanished descendant (see above) against other + * writers (block acceptance, double-spend replacement) with backoff; once + * retries are exhausted it throws, leaving the rows for the next expiration + * scan. + */ +export const archiveMempoolTransactions = async ( + expiries: MempoolTransactionExpiry[] +): Promise => { + if (expiries.length === 0) { + return []; + } + const sorted = [...expiries].sort( + (a, b) => + a.nodeInternalId - b.nodeInternalId || + a.transactionInternalId - b.transactionInternalId + ); + const nodeInternalIds = sorted.map((expiry) => expiry.nodeInternalId); + const transactionInternalIds = sorted.map( + (expiry) => expiry.transactionInternalId + ); + const archiveOnce = async () => { + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query('BEGIN;'); + await client.query('SELECT pg_advisory_xact_lock($1::bigint);', [ + expiryAdvisoryLockKey, + ]); + await client.query( + /* sql */ ` +SELECT 1 + FROM node_transaction + JOIN unnest($1::integer[], $2::bigint[]) + AS expiry (node_internal_id, transaction_internal_id) + ON node_transaction.node_internal_id = expiry.node_internal_id + AND node_transaction.transaction_internal_id = expiry.transaction_internal_id + ORDER BY node_transaction.node_internal_id, node_transaction.transaction_internal_id + FOR UPDATE OF node_transaction; `, - [ - nodeInternalIds, - transactionInternalIds, - sorted.map((expiry) => expiry.replacedAt.toISOString()), - sorted.map((expiry) => expiry.validatedAt?.toISOString() ?? null), - ] + [nodeInternalIds, transactionInternalIds] ); + const result = await client.query<{ + archived: string | null; + complete: boolean; + }>(mempoolExpiryArchivalStatement, [ + nodeInternalIds, + transactionInternalIds, + sorted.map((expiry) => expiry.replacedAt.toISOString()), + sorted.map((expiry) => expiry.validatedAt?.toISOString() ?? null), + ]); if (result.rows[0]?.complete !== true) { // eslint-disable-next-line functional/no-throw-statement throw Object.assign( diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 9957d560..2a2aafad 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -855,6 +855,32 @@ const waitForConfirmedMempoolArchive = async ( return waitForConfirmedMempoolArchive(remainingAttempts - 1); }; +const mempoolPollingAttempts = 50; +const mempoolPollingIntervalMs = 100; +const getMempoolTransactionHashes = async () => + ( + await client.query<{ hash: string }>( + /* sql */ `SELECT encode(hash, 'hex') AS hash FROM node_transaction JOIN transaction ON node_transaction.transaction_internal_id = transaction.internal_id ORDER BY hash ASC;` + ) + ).rows.map((row) => row.hash); +/** + * Poll node_transaction until its hashes (as listed by + * {@link getMempoolTransactionHashes}) satisfy `expected`: the agent saves + * transactions received from peers asynchronously. Returns the last hashes + * read. + */ +const waitForMempoolTransactionHashes = async ( + expected: (hashes: string[]) => boolean, + remainingAttempts = mempoolPollingAttempts +): Promise => { + const hashes = await getMempoolTransactionHashes(); + if (expected(hashes) || remainingAttempts === 0) { + return hashes; + } + await sleep(mempoolPollingIntervalMs); + return waitForMempoolTransactionHashes(expected, remainingAttempts - 1); +}; + test.serial( '[e2e] ignores inbound transactions before initial sync is complete', async (t) => { @@ -2125,6 +2151,14 @@ test.serial( const [, tx1] = tipA[0]!.transactions; peers.node1.sendMessage(new peers.node1.messages.Transaction(tx1)); logger.debug(`node1: sent tipA[0] transaction 0: ${tx1!.hash}`); + // the block must arrive after the transaction is saved for it to be known + t.true( + ( + await waitForMempoolTransactionHashes((hashes) => + hashes.includes(tx1!.hash) + ) + ).includes(tx1!.hash) + ); newBlocks('node1', [tipA[0]!]); newBlocks('node2', [tipB[0]!]); newBlocks('node3', [tipB[0]!]); @@ -2482,20 +2516,21 @@ test.serial( peers.node1.sendMessage(new peers.node1.messages.Transaction(tx2)); logger.debug(`node1: sent tx2: ${tx2!.hash}`); peers.node1.sendMessage(new peers.node1.messages.Transaction(tx3)); - logger.debug(`node1: sent tx3: ${tx2!.hash}`); - const delay = 100; - await sleep(delay); - const mempool1 = await client.query<{ encode: string }>( - /* sql */ `SELECT encode(hash, 'hex') FROM node_transaction JOIN transaction ON node_transaction.transaction_internal_id = transaction.internal_id ORDER BY hash ASC;` - ); - t.deepEqual(mempool1.rows, [ - { encode: tx1!.hash }, - { encode: tx3!.hash }, - { encode: tx2!.hash }, - { encode: chipnetCashTokensTxHash }, - { encode: halTxSpent }, - { encode: halTxHash }, - ]); + logger.debug(`node1: sent tx3: ${tx3!.hash}`); + const expectedMempool = [ + tx1!.hash, + tx3!.hash, + tx2!.hash, + chipnetCashTokensTxHash, + halTxSpent, + halTxHash, + ]; + t.deepEqual( + await waitForMempoolTransactionHashes( + (hashes) => hashes.join() === expectedMempool.join() + ), + expectedMempool + ); newBlocks('node1', [tipA[161]!]); newBlocks('node2', [tipB[161]!]); newBlocks('node3', [tipA[161]!]); @@ -2507,13 +2542,10 @@ test.serial( await waitForStdout( /Saved new block – height:\s+3162[^\n]+nodes: node1, node3/u ); - const mempool2 = await client.query<{ encode: string }>( - /* sql */ `SELECT encode(hash, 'hex') FROM node_transaction JOIN transaction ON node_transaction.transaction_internal_id = transaction.internal_id ORDER BY hash ASC;` - ); - t.deepEqual(mempool2.rows, [ - { encode: chipnetCashTokensTxHash }, - { encode: halTxSpent }, - { encode: halTxHash }, + t.deepEqual(await getMempoolTransactionHashes(), [ + chipnetCashTokensTxHash, + halTxSpent, + halTxHash, ]); t.pass(); } diff --git a/src/e2e/mempool-expiry-concurrency.spec.ts b/src/e2e/mempool-expiry-concurrency.spec.ts index 3084452a..ce4695ff 100644 --- a/src/e2e/mempool-expiry-concurrency.spec.ts +++ b/src/e2e/mempool-expiry-concurrency.spec.ts @@ -6,7 +6,7 @@ import baseTest from 'ava'; import type * as database from '../db.js'; -// cspell:words lpad xact +// cspell:words autovacuum lpad xact const test = baseTest as TestFn<{ db: typeof database; schema: string }>; const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; @@ -69,8 +69,19 @@ test.after.always(async (t) => { await t.context.db.pool.end(); }); -const chains = 60; -const depth = 40; +/** The expiry queue's batch size (`expiryBatchSize` in db.ts). */ +const expiryQueueBatchSize = 500; +interface MempoolShape { + chains: number; + depth: number; +} +/* + * Big enough for several expiry queue batches and many concurrent single-row + * expiries of related rows; small enough that the single-row test, which + * serializes one transaction per row, stays fast on a slow CI runner. + */ +const mempoolShape: MempoolShape = { chains: 20, depth: 15 }; +const mempoolRowCount = ({ chains, depth }: MempoolShape) => chains * depth * 2; /** * `chains` chains of `depth` mempool transactions (each spends output 0 of the @@ -80,7 +91,8 @@ const depth = 40; */ const createStaleMempool = async ( db: typeof database, - validatedAt = `'2026-01-01'::timestamp` + validatedAt = `'2026-01-01'::timestamp`, + { chains, depth } = mempoolShape ) => { await db.pool.query(/* sql */ ` TRUNCATE node, transaction, output, input, node_transaction, @@ -149,7 +161,7 @@ const expectAllArchived = async ( ); t.deepEqual(await countRows(t.context.db), { current: 0, - history: chains * depth * 2, + history: mempoolRowCount(mempoolShape), }); const replacedAt = await t.context.db.pool.query<{ replaced: string }>( /* sql */ `SELECT DISTINCT replaced_at::text AS replaced FROM node_transaction_history;` @@ -157,8 +169,12 @@ const expectAllArchived = async ( t.deepEqual(replacedAt.rows, [{ replaced }]); }; -const staleExpiries = async (db: typeof database, validatedAt?: string) => { - await createStaleMempool(db, validatedAt); +const staleExpiries = async ( + db: typeof database, + validatedAt?: string, + shape = mempoolShape +) => { + await createStaleMempool(db, validatedAt, shape); const expiring = await db.getMempoolTransactionsExpiringBefore({ expirationMs: 3_600_000, expiresBefore: new Date('2026-01-10T00:00:00.000Z'), @@ -183,7 +199,7 @@ test.serial( async (t) => { const { db } = t.context; const expiries = await staleExpiries(db); - t.is(expiries.length, chains * depth * 2); + t.is(expiries.length, mempoolRowCount(mempoolShape)); const errors: unknown[] = []; await Promise.all( expiries.map(async (expiry) => @@ -270,6 +286,92 @@ test.serial( } ); +/* eslint-disable @typescript-eslint/naming-convention */ +interface PlanNode { + 'Actual Loops': number; + 'Actual Rows': number; + 'Node Type': string; + Plans?: PlanNode[]; + 'Relation Name'?: string; + 'Rows Removed by Filter'?: number; + 'Rows Removed by Index Recheck'?: number; +} +/* eslint-enable @typescript-eslint/naming-convention */ +const rowsReadPerLoop = (node: PlanNode) => + node['Actual Rows'] + + (node['Rows Removed by Filter'] ?? 0) + + (node['Rows Removed by Index Recheck'] ?? 0); +/** + * Rows of `relation` read by the scans of an EXPLAIN ANALYZE plan: those + * emitted plus those discarded by the scan's filter or index recheck (all + * reported per loop). + */ +const rowsScanned = (node: PlanNode, relation: string): number => + (node['Relation Name'] === relation && node['Node Type'].endsWith('Scan') + ? rowsReadPerLoop(node) * node['Actual Loops'] + : 0) + + (node.Plans ?? []).reduce( + (total, child) => total + rowsScanned(child, relation), + 0 + ); + +/* + * Regression: with node_transaction statistics taken when the mempool was + * nearly empty (as autovacuum may analyze it just after a block), Postgres + * joined each walked descendant to node_transaction by node alone, reading the + * node's whole mempool per walked row: archiving a stale mempool took time + * quadratic in its size (here ~485,000 node_transaction rows read rather than + * ~20,000; tens of seconds per batch on CI). + */ +test.serial( + '[e2e] archiving a stale mempool reads each node_transaction row a few times, also with outdated statistics', + async (t) => { + const { db } = t.context; + // larger, so the quadratic plan reads far more rows than the bound below + const largeMempool = { chains: 60, depth: 40 }; + const mempoolRows = mempoolRowCount(largeMempool); + // statistics of a nearly empty mempool, then the stale mempool + await createStaleMempool(db, undefined, largeMempool); + await db.pool.query(/* sql */ ` +DELETE FROM node_transaction WHERE transaction_internal_id > 3; +ANALYZE node_transaction;`); + // one expiry queue batch, including transaction 1: the ancestor of all + const expiries = (await staleExpiries(db, undefined, largeMempool)) + .sort((a, b) => a.transactionInternalId - b.transactionInternalId) + .slice(0, expiryQueueBatchSize); + const client = await db.pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query('BEGIN;'); + const explained = await client.query<{ + // eslint-disable-next-line @typescript-eslint/naming-convention + 'QUERY PLAN': [{ Plan: PlanNode }]; + }>( + `EXPLAIN (ANALYZE, FORMAT JSON) ${db.mempoolExpiryArchivalStatement}`, + [ + expiries.map((expiry) => expiry.nodeInternalId), + expiries.map((expiry) => expiry.transactionInternalId), + expiries.map((expiry) => expiry.replacedAt.toISOString()), + expiries.map(() => null), + ] + ); + const scanned = rowsScanned( + explained.rows[0]!['QUERY PLAN'][0].Plan, + 'node_transaction' + ); + t.log(`node_transaction rows scanned: ${scanned}`); + t.true(scanned <= mempoolRows * 10, `scanned ${scanned} rows`); + const archived = await client.query<{ count: string }>( + /* sql */ `SELECT count(*)::text FROM node_transaction_history;` + ); + t.is(Number(archived.rows[0]!.count), mempoolRows); + } finally { + await client.query('ROLLBACK;'); + client.release(); + } + } +); + /* * Expire transaction 1 of a two-transaction mempool for alpha (2 spends 1, and * `prepare` runs first) while another connection's transaction, `blocker`,