diff --git a/.cspell.json b/.cspell.json index 9182b0da..ba35d020 100644 --- a/.cspell.json +++ b/.cspell.json @@ -71,6 +71,7 @@ "peerready", "peersendheaders", "pgadmin", + "PGCOPY", "pghero", "pino", "pkey", @@ -97,12 +98,17 @@ "sendheaders", "statefulset", "statefulsets", + "subquery", "symlinks", "templating", "txindex", "unintuitive", + "unnest", "unpruned", "upserting", + "UTXO", + "UTXOs", + "varint", "verack", "xprivkey", "xpubkey" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index efa97ce1..95ef46d1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,6 +4,13 @@ on: [push, pull_request] jobs: e2e-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + write-path: [sql, copy] + name: e2e-tests (write path ${{ matrix.write-path }}) + env: + CHAINGRAPH_WRITE_PATH: ${{ matrix.write-path }} services: postgres: image: postgres:14 @@ -20,36 +27,36 @@ jobs: - 5432:5432 steps: - name: Check out repository code - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: submodules: 'true' - name: Install Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v7 with: node-version: '18' - run: yarn install --immutable --immutable-cache - run: yarn test - run: yarn cov:lcov - name: Upload test coverage - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} - name: Save E2E test logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v7 if: always() with: - name: e2e-test-log + name: e2e-test-log-${{ matrix.write-path }} path: data/chaingraph/log-e2e.ndjson check-yarn-cache: runs-on: ubuntu-latest steps: - name: Check out repository code - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: submodules: 'true' - name: Install Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v7 with: node-version: '18' - run: yarn install --immutable --immutable-cache --check-cache diff --git a/.github/workflows/image-agent.yaml b/.github/workflows/image-agent.yaml index 0791b7d5..292c7ad3 100644 --- a/.github/workflows/image-agent.yaml +++ b/.github/workflows/image-agent.yaml @@ -17,9 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v4 - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: submodules: 'true' - name: Get tag @@ -27,13 +27,13 @@ jobs: echo "::set-output name=commit::$(git rev-parse --short HEAD)" id: get_tag - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Build, tag, push id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v7 with: context: . file: images/agent/Dockerfile diff --git a/.github/workflows/image-bchn.yaml b/.github/workflows/image-bchn.yaml index 625d545d..ceb80bf5 100644 --- a/.github/workflows/image-bchn.yaml +++ b/.github/workflows/image-bchn.yaml @@ -12,21 +12,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v4 - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Get version run: | echo "::set-output name=version::$(grep -F 'ENV BITCOIN_VERSION' images/bitcoin-cash-node/Dockerfile | grep -oE '[0-9.]+')" id: get_version - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Build, tag, push id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v7 with: context: images/bitcoin-cash-node push: true diff --git a/.github/workflows/image-hasura.yaml b/.github/workflows/image-hasura.yaml index 5a9c34a4..f61b08fd 100644 --- a/.github/workflows/image-hasura.yaml +++ b/.github/workflows/image-hasura.yaml @@ -12,21 +12,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v4 - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Get tag run: | echo "::set-output name=commit::$(git rev-parse --short HEAD)" id: get_tag - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Build, tag, push id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v7 with: context: images/hasura push: true diff --git a/.github/workflows/release-chart.yaml b/.github/workflows/release-chart.yaml index e07c20fe..27dfbd4d 100644 --- a/.github/workflows/release-chart.yaml +++ b/.github/workflows/release-chart.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: fetch-depth: 0 - name: Configure Git @@ -18,7 +18,7 @@ jobs: git config user.name "$GITHUB_ACTOR" git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - name: Install Helm - uses: azure/setup-helm@v3 + uses: azure/setup-helm@v5 - name: Run chart-releaser uses: helm/chart-releaser-action@v1.5.0 with: diff --git a/.gitignore b/.gitignore index aa314efb..249d16d4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ package-lock.json data .env client/generated +.scratch diff --git a/.yarn b/.yarn index a7a702dd..d33cb1f7 160000 --- a/.yarn +++ b/.yarn @@ -1 +1 @@ -Subproject commit a7a702dde60fc7eb627f4e7a655d6063590a4e69 +Subproject commit d33cb1f7cfb2386378c6a9c93564e79153417fcb diff --git a/charts/chaingraph/templates/agent.yaml b/charts/chaingraph/templates/agent.yaml index 5b1c17fc..86eaaa24 100644 --- a/charts/chaingraph/templates/agent.yaml +++ b/charts/chaingraph/templates/agent.yaml @@ -8,6 +8,14 @@ metadata: {{- include "chaingraph.labels" . | nindent 4 }} spec: replicas: 1 + # The agent is a singleton writer: never run two concurrently. Recreate + # terminates the old pod before starting its replacement, at the cost of a + # brief indexing gap (the agent catches up on restart). + strategy: + type: Recreate + # Explicit null clears the server-defaulted rollingUpdate block left by + # previous RollingUpdate deployments (Helm three-way merge). + rollingUpdate: null selector: matchLabels: app: chaingraph-agent @@ -41,6 +49,8 @@ spec: value: {{ .Values.agent.synchronousCommit | quote }} - name: CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB value: {{ .Values.agent.blockBufferTargetSizeMb | quote }} + - name: CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE + value: {{ .Values.agent.repairBlocksBatchSize | quote }} - name: CHAINGRAPH_GENESIS_BLOCKS value: {{ .Values.agent.genesisBlocks | quote }} - name: CHAINGRAPH_TRUSTED_NODES @@ -68,13 +78,17 @@ spec: timeoutSeconds: 15 successThreshold: 1 failureThreshold: 3 + # Readiness is tight so rollouts report Ready seconds after the + # internal API is up. Liveness (above) stays coarse on purpose: a + # CPU-bound catch-up can block the event loop (and /health-check), + # and a twitchy liveness probe would kill the agent mid-recovery. readinessProbe: httpGet: path: /health-check port: {{ .Values.agent.port }} - initialDelaySeconds: 30 - periodSeconds: 60 - timeoutSeconds: 15 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 resources: {} diff --git a/charts/chaingraph/values.yaml b/charts/chaingraph/values.yaml index 88871270..941e8193 100644 --- a/charts/chaingraph/values.yaml +++ b/charts/chaingraph/values.yaml @@ -18,12 +18,15 @@ agent: # The maximum number of connections the agent should maintain to the database. # For best performance, this should be set to the number of CPUs available to Postgres. If not set, Chaingraph will assume that Postgres is running on hardware equivalent to its own. (This is ideal if Postgres is running on either the same machine or an equivalent one from a homogenous Kubernetes node pool.) maxConnections: '' -# If set to false, the Postgres database will be configured to use "synchronous_commit = off" during initial sync. -# In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default. + # If set to false, the Postgres database will be configured to use "synchronous_commit = off" during initial sync. + # In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default. synchronousCommit: true # The target size (in MB) of the buffer which holds downloaded blocks waiting to be saved to the database. This primarily affects memory usage during the initial chain sync. # For best performance, this should be around `maxConnections * maximum block size`, while leaving enough memory available to the host machine. If left unset (recommended), Chaingraph will measure free memory at startup and attempt to select a reasonable value. blockBufferTargetSizeMb: '' + # Self-healing for https://github.com/bitauth/chaingraph/issues/74: after initial sync, the agent scans saved blocks in batches to find blocks whose linked transactions don't match the saved block size, then repairs those blocks by re-requesting them from trusted nodes. + # Set to 0 to disable the startup repair task. Completed scans are recorded per node in the incomplete_block_audit table, so later startups only audit blocks saved since the last completed scan; delete its rows to force a full re-audit. + repairBlocksBatchSize: 10000 # A mapping of network magic bytes to hex-encoded genesis blocks. # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated. # E.g. CHAINGRAPH_GENESIS_BLOCKS=e3e1f3e8:rawblockhex,deadbeef:rawblockhex diff --git a/defaults.env b/defaults.env index f6f87fca..79925c43 100644 --- a/defaults.env +++ b/defaults.env @@ -12,10 +12,43 @@ CHAINGRAPH_POSTGRES_MAX_CONNECTIONS= # In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default. CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT=true +# How block data is written to Postgres: `sql` sends each block as multi-row `INSERT … VALUES` statements; `copy` streams the rows with binary `COPY` into per-connection temporary staging tables and inserts from there (same constraints and `ON CONFLICT DO NOTHING` semantics, much less SQL text to build, send, parse and plan). +CHAINGRAPH_WRITE_PATH=sql + # The target size (in MB) of the buffer which holds downloaded blocks waiting to be saved to the database. This primarily affects memory usage during the initial chain sync. # For best performance, this should be around `CHAINGRAPH_POSTGRES_MAX_CONNECTIONS * maximum block size`, while leaving enough memory available to the host machine. If not set, Chaingraph will measure free memory at startup and attempt to select a reasonable value. CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB= +# Self-healing for https://github.com/bitauth/chaingraph/issues/74: after +# initial sync, the agent scans saved blocks in batches to find blocks whose +# linked transactions don't match the saved block size, then repairs those +# blocks by re-requesting them from trusted nodes. Set to 0 to disable the +# startup repair task. Completed scans are recorded per node in the +# incomplete_block_audit table, so later startups only audit blocks saved since +# the last completed scan; delete its rows to force a full re-audit. +CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE=10000 + +# BCHN expires transactions from the mempool after 336 hours but does not announce +# those expirations. At initial sync and periodically thereafter the agent scans for +# node_transaction rows expiring soon and schedules each row to be archived to +# node_transaction_history at its exact expiry time. The existing mempool +# invalidation cascade archives same-node descendants. See bitauth/chaingraph#73. +CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS=1209600000 +CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS=3600000 + +# BCHN does not re-announce transactions that a re-organization returns to its +# mempool. When enabled, the agent sends each trusted node a BIP35 `mempool` +# request after mempool tracking is enabled, after the node reconnects, and +# after each re-organization reported by that node, so the answering `inv` +# re-records `node_transaction` rows. Replies only add rows. Requests to each +# node are rate-limited to one per CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS; +# requests during that window are coalesced into one request when it ends. +# The node must answer `mempool` requests: BCHN does when NODE_BLOOM is enabled +# (`-peerbloomfilters`, the default) or the agent has the `mempool` permission +# (e.g. `-whitelist=mempool@`). See bitauth/chaingraph#81. +CHAINGRAPH_MEMPOOL_RESYNC=true +CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS=30000 + # A mapping of network magic bytes to hex-encoded genesis blocks. # Format: `NETWORK_MAGIC:RAW_GENESIS_BLOCK_HEX`, comma separated. # E.g. CHAINGRAPH_GENESIS_BLOCKS=e3e1f3e8:rawblockhex,deadbeef:rawblockhex diff --git a/images/agent/Dockerfile b/images/agent/Dockerfile index f0a0d499..67a845bf 100644 --- a/images/agent/Dockerfile +++ b/images/agent/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-alpine AS build-stage +FROM node:24-alpine AS build-stage WORKDIR /chaingraph COPY package.json yarn.lock .yarnrc.yml ./ COPY .yarn .yarn @@ -10,7 +10,7 @@ RUN yarn build # Use Yarn plugin-production-install to copy only production dependencies (https://gitlab.com/Larry1123/yarn-contrib/-/tree/master/packages/plugin-production-install) RUN yarn prod-install ./production-dependencies -FROM node:18-alpine AS production +FROM node:24-alpine AS production WORKDIR /chaingraph COPY --from=build-stage /chaingraph/production-dependencies ./ COPY --from=build-stage /chaingraph/defaults.env ./ diff --git a/images/bitcoin-cash-node/Dockerfile b/images/bitcoin-cash-node/Dockerfile index 55008df7..b84ac54b 100644 --- a/images/bitcoin-cash-node/Dockerfile +++ b/images/bitcoin-cash-node/Dockerfile @@ -7,9 +7,9 @@ RUN set -ex \ && apt-get install -qq --no-install-recommends ca-certificates gosu wget \ && rm -rf /var/lib/apt/lists/* -ENV BITCOIN_VERSION 28.0.2 -ENV BITCOIN_URL https://download.bitcoincashnode.org/misc/builds/upgrade12_temp/linux/bitcoin-cash-node-28.0.2-x86_64-linux-gnu.tar.gz -ENV BITCOIN_SHA256 140b44fd76a4f9428354bfbec4800d58fd39fb723320e761a035f15c2dd43596 +ENV BITCOIN_VERSION 29.0.0 +ENV BITCOIN_URL https://github.com/bitcoin-cash-node/bitcoin-cash-node/releases/download/v29.0.0/bitcoin-cash-node-29.0.0-x86_64-linux-gnu.tar.gz +ENV BITCOIN_SHA256 6125d1cbecc1db476f2b6b7b91da5acde92d2311b8e738124e3db64ca84b33e1 # install bitcoin binaries RUN set -ex \ diff --git a/images/hasura/Dockerfile b/images/hasura/Dockerfile index f59c2cd3..d345f0ec 100644 --- a/images/hasura/Dockerfile +++ b/images/hasura/Dockerfile @@ -1,5 +1,10 @@ -FROM hasura/graphql-engine:v2.16.1.cli-migrations-v3 +# Multi-arch tag; digest of the manifest list as of 2026-07-20: +# sha256:da573f6420bfbc330023dd4b816c1e6fcb50ab6a00fd3e1d8c70088532d5fa7f +# NOTE: first startup against an existing database performs a ONE-WAY +# metadata catalog upgrade (47 -> current). Snapshot/backup hdb_catalog +# before deploying to an instance you may want to roll back. +FROM hasura/graphql-engine:v2.49.5.cli-migrations-v3 COPY hasura-data /hasura-data diff --git a/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/down.sql b/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/down.sql new file mode 100644 index 00000000..6def7fc0 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/down.sql @@ -0,0 +1,2 @@ +DROP TRIGGER IF EXISTS trigger_public_node_transaction_history_insert ON node_transaction_history; +DROP FUNCTION IF EXISTS trigger_node_transaction_history_insert(); diff --git a/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql b/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql new file mode 100644 index 00000000..b837b5bc --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql @@ -0,0 +1,105 @@ +CREATE OR REPLACE FUNCTION trigger_node_transaction_history_insert() RETURNS trigger + LANGUAGE plpgsql +AS $$ +BEGIN + /* + * The recursive CTE archives the full descendant set in one pass. The + * archive INSERT below re-fires this trigger, but that re-entry should return + * immediately rather than attempting another cascade. + */ + IF current_setting('chaingraph.suppress_mempool_descendant_cascade', true) = 'on' THEN + RETURN NULL; + END IF; + + -- Confirmations and empty batches do not invalidate descendants. + IF NOT EXISTS (SELECT 1 FROM new_table WHERE replaced_at IS NOT NULL) THEN + RETURN NULL; + END IF; + + PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'on', true); + + BEGIN + /* + * If another session deletes matching descendants before this DELETE + * reaches them, this archives zero rows. Suppression still prevents empty + * self-reentry. + */ + WITH RECURSIVE descendant_transactions AS ( + -- Seed: mempool transactions spending outputs of newly replaced parents. + SELECT nt.node_internal_id, + nt.transaction_internal_id, + nt.validated_at, + replaced_parents.replaced_at + FROM new_table replaced_parents + INNER JOIN transaction parent_transaction + ON parent_transaction.internal_id = replaced_parents.transaction_internal_id + INNER JOIN output parent_output + ON parent_output.transaction_hash = parent_transaction.hash + INNER JOIN input + ON input.outpoint_transaction_hash = parent_output.transaction_hash + AND input.outpoint_index = parent_output.output_index + INNER JOIN node_transaction nt + ON nt.transaction_internal_id = input.transaction_internal_id + AND nt.node_internal_id = replaced_parents.node_internal_id + WHERE replaced_parents.replaced_at IS NOT NULL + + UNION + + -- Recursive step: mempool transactions spending outputs of descendants. + SELECT child_nt.node_internal_id, + child_nt.transaction_internal_id, + child_nt.validated_at, + parent_descendants.replaced_at + FROM descendant_transactions parent_descendants + INNER JOIN transaction parent_transaction + ON parent_transaction.internal_id = parent_descendants.transaction_internal_id + INNER JOIN output parent_output + ON parent_output.transaction_hash = parent_transaction.hash + INNER JOIN input + ON input.outpoint_transaction_hash = parent_output.transaction_hash + AND input.outpoint_index = parent_output.output_index + INNER JOIN node_transaction child_nt + ON child_nt.transaction_internal_id = input.transaction_internal_id + AND child_nt.node_internal_id = parent_descendants.node_internal_id + ), + descendants AS ( + -- If reachable through multiple replaced parents, use earliest invalidation. + SELECT node_internal_id, + transaction_internal_id, + validated_at, + MIN(replaced_at) AS replaced_at + FROM descendant_transactions + GROUP BY node_internal_id, transaction_internal_id, validated_at + ), + deleted_descendants AS ( + DELETE FROM node_transaction + USING descendants + WHERE node_transaction.node_internal_id = descendants.node_internal_id + AND node_transaction.transaction_internal_id = descendants.transaction_internal_id + RETURNING node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + node_transaction.validated_at, + descendants.replaced_at + ) + INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at + FROM deleted_descendants; + EXCEPTION WHEN OTHERS THEN + PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'off', true); + RAISE; + END; + + PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'off', true); + RETURN NULL; +END; +$$; + +CREATE TRIGGER trigger_public_node_transaction_history_insert + AFTER INSERT ON node_transaction_history + REFERENCING NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE FUNCTION trigger_node_transaction_history_insert(); +COMMENT ON TRIGGER trigger_public_node_transaction_history_insert ON node_transaction_history + IS 'Cascades mempool invalidation recursively: when a node_transaction is archived to history with replaced_at set, all same-node descendants still present in node_transaction are archived with a deterministic replaced_at timestamp.'; + +-- disabled until initial sync is complete (when mempool transactions begin to be accepted) +ALTER TABLE node_transaction_history DISABLE TRIGGER trigger_public_node_transaction_history_insert; diff --git a/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/down.sql b/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/down.sql new file mode 100644 index 00000000..918fd2db --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/down.sql @@ -0,0 +1,2 @@ +-- This migration archives orphaned mempool descendants into +-- node_transaction_history. The data move is intentionally not reversible. diff --git a/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/up.sql b/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/up.sql new file mode 100644 index 00000000..d90a6273 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778158619747_backfill_orphan_mempool_descendants/up.sql @@ -0,0 +1,88 @@ +DO $$ +DECLARE + previous_suppression text := current_setting('chaingraph.suppress_mempool_descendant_cascade', true); +BEGIN + /* + * The backfill archives the full existing orphan set itself. If the + * node_transaction_history trigger is enabled while this migration runs, + * suppress its re-entry from the archive INSERT below. + */ + PERFORM set_config('chaingraph.suppress_mempool_descendant_cascade', 'on', true); + + BEGIN + WITH RECURSIVE orphan_transactions AS ( + -- Seed: mempool transactions whose parent was already archived as replaced. + SELECT nt.node_internal_id, + nt.transaction_internal_id, + nt.validated_at, + nth.replaced_at + FROM node_transaction nt + INNER JOIN input child_input + ON child_input.transaction_internal_id = nt.transaction_internal_id + INNER JOIN transaction parent_transaction + ON parent_transaction.hash = child_input.outpoint_transaction_hash + INNER JOIN output parent_output + ON parent_output.transaction_hash = parent_transaction.hash + AND parent_output.output_index = child_input.outpoint_index + INNER JOIN node_transaction_history nth + ON nth.transaction_internal_id = parent_transaction.internal_id + AND nth.node_internal_id = nt.node_internal_id + WHERE nth.replaced_at IS NOT NULL + + UNION + + -- Recursive step: mempool transactions spending outputs of known orphans. + SELECT child_nt.node_internal_id, + child_nt.transaction_internal_id, + child_nt.validated_at, + parent_orphans.replaced_at + FROM orphan_transactions parent_orphans + INNER JOIN transaction parent_transaction + ON parent_transaction.internal_id = parent_orphans.transaction_internal_id + INNER JOIN output parent_output + ON parent_output.transaction_hash = parent_transaction.hash + INNER JOIN input child_input + ON child_input.outpoint_transaction_hash = parent_output.transaction_hash + AND child_input.outpoint_index = parent_output.output_index + INNER JOIN node_transaction child_nt + ON child_nt.transaction_internal_id = child_input.transaction_internal_id + AND child_nt.node_internal_id = parent_orphans.node_internal_id + ), + orphans AS ( + -- If reachable through multiple replaced parents, use earliest invalidation. + SELECT node_internal_id, + transaction_internal_id, + validated_at, + MIN(replaced_at) AS replaced_at + FROM orphan_transactions + GROUP BY node_internal_id, transaction_internal_id, validated_at + ), + deleted_orphans AS ( + DELETE FROM node_transaction + USING orphans + WHERE node_transaction.node_internal_id = orphans.node_internal_id + AND node_transaction.transaction_internal_id = orphans.transaction_internal_id + RETURNING node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + node_transaction.validated_at, + orphans.replaced_at + ) + INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at + FROM deleted_orphans; + EXCEPTION WHEN OTHERS THEN + PERFORM set_config( + 'chaingraph.suppress_mempool_descendant_cascade', + COALESCE(previous_suppression, 'off'), + true + ); + RAISE; + END; + + PERFORM set_config( + 'chaingraph.suppress_mempool_descendant_cascade', + COALESCE(previous_suppression, 'off'), + true + ); +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/down.sql b/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/down.sql new file mode 100644 index 00000000..6eb242d3 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/down.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE FUNCTION encode_block(block_row block) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + transactions CURSOR FOR SELECT transaction.* FROM transaction + INNER JOIN block_transaction ON transaction.internal_id = block_transaction.transaction_internal_id + WHERE block_transaction.block_internal_id = block_row.internal_id + ORDER BY block_transaction.transaction_index ASC; + encoded_block bytea := encode_block_header(block_row) || encode_compact_uint(COUNT(transactions)); +BEGIN + FOR transaction_row IN transactions + LOOP + encoded_block := encoded_block || + encode_transaction(ROW( + transaction_row.internal_id, + transaction_row.hash, + transaction_row.version, + transaction_row.locktime, + transaction_row.size_bytes, + transaction_row.is_coinbase)::transaction + ); + END LOOP; + RETURN encoded_block; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/up.sql b/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/up.sql new file mode 100644 index 00000000..4e572eee --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778351597200_fix_block_encoded_transaction_count/up.sql @@ -0,0 +1,30 @@ +CREATE OR REPLACE FUNCTION encode_block(block_row block) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + transactions CURSOR FOR SELECT transaction.* FROM transaction + INNER JOIN block_transaction ON transaction.internal_id = block_transaction.transaction_internal_id + WHERE block_transaction.block_internal_id = block_row.internal_id + ORDER BY block_transaction.transaction_index ASC; + encoded_block bytea := encode_block_header(block_row); + transaction_count bigint := 0; +BEGIN + SELECT COUNT(*) INTO transaction_count + FROM block_transaction + WHERE block_transaction.block_internal_id = block_row.internal_id; + encoded_block := encoded_block || encode_compact_uint(transaction_count); + FOR transaction_row IN transactions + LOOP + encoded_block := encoded_block || + encode_transaction(ROW( + transaction_row.internal_id, + transaction_row.hash, + transaction_row.version, + transaction_row.locktime, + transaction_row.size_bytes, + transaction_row.is_coinbase)::transaction + ); + END LOOP; + RETURN encoded_block; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/down.sql b/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/down.sql new file mode 100644 index 00000000..7608b37e --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/down.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE FUNCTION transaction_data_carrier_outputs(transaction_row transaction) RETURNS SETOF output + LANGUAGE sql IMMUTABLE +AS $$ + SELECT * FROM output WHERE transaction_hash = $1.hash AND (value_satoshis = 0 OR get_byte(locking_bytecode, 0) = 106); +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql b/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql new file mode 100644 index 00000000..b1cdd137 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE FUNCTION transaction_data_carrier_outputs(transaction_row transaction) RETURNS SETOF output + LANGUAGE sql IMMUTABLE +AS $$ + SELECT * FROM output WHERE transaction_hash = $1.hash AND (value_satoshis = 0 OR (octet_length(locking_bytecode) > 0 AND get_byte(locking_bytecode, 0) = 106)); +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/down.sql b/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/down.sql new file mode 100644 index 00000000..1562de57 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/down.sql @@ -0,0 +1,59 @@ +CREATE OR REPLACE FUNCTION transaction_input_value_satoshis(transaction_row transaction) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(inputs.input_value_satoshis)::bigint FROM ( + SELECT input_value_satoshis (input) FROM input WHERE transaction_internal_id = transaction_row.internal_id + ) as "inputs" +$$; + +CREATE OR REPLACE FUNCTION block_input_count(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_input_count)::bigint FROM ( + SELECT transaction_input_count (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_input_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_input_value_satoshis)::bigint FROM ( + SELECT transaction_input_value_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_output_count(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_output_count)::bigint FROM ( + SELECT transaction_output_count (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_output_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_output_value_satoshis)::bigint FROM ( + SELECT transaction_output_value_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION transaction_fee_satoshis(transaction_row transaction) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT transaction_input_value_satoshis(transaction_row) - transaction_output_value_satoshis(transaction_row) +$$; + +CREATE OR REPLACE FUNCTION block_fee_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT SUM(transactions.transaction_fee_satoshis)::bigint FROM ( + SELECT transaction_fee_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id ) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_generated_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT (block_output_value_satoshis(block) - block_input_value_satoshis(block))::bigint FROM block WHERE internal_id = block_row.internal_id; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/up.sql b/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/up.sql new file mode 100644 index 00000000..b6045378 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778429124205_fix_coinbase_only_value_aggregates/up.sql @@ -0,0 +1,62 @@ +CREATE OR REPLACE FUNCTION transaction_input_value_satoshis(transaction_row transaction) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(inputs.input_value_satoshis), 0)::bigint FROM ( + SELECT input_value_satoshis (input) FROM input WHERE transaction_internal_id = transaction_row.internal_id + ) as "inputs" +$$; + +CREATE OR REPLACE FUNCTION block_input_count(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_input_count), 0)::bigint FROM ( + SELECT transaction_input_count (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_input_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_input_value_satoshis), 0)::bigint FROM ( + SELECT transaction_input_value_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_output_count(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_output_count), 0)::bigint FROM ( + SELECT transaction_output_count (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_output_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_output_value_satoshis), 0)::bigint FROM ( + SELECT transaction_output_value_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION transaction_fee_satoshis(transaction_row transaction) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT CASE + WHEN transaction_row.is_coinbase THEN 0 + ELSE transaction_input_value_satoshis(transaction_row) - transaction_output_value_satoshis(transaction_row) + END +$$; + +CREATE OR REPLACE FUNCTION block_fee_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT COALESCE(SUM(transactions.transaction_fee_satoshis), 0)::bigint FROM ( + SELECT transaction_fee_satoshis (transaction) FROM transaction WHERE internal_id IN (SELECT transaction_internal_id from block_transaction WHERE block_internal_id = block_row.internal_id ) + ) as "transactions" +$$; + +CREATE OR REPLACE FUNCTION block_generated_value_satoshis(block_row block) RETURNS bigint + LANGUAGE sql IMMUTABLE +AS $$ + SELECT (block_output_value_satoshis(block) - block_input_value_satoshis(block))::bigint FROM block WHERE internal_id = block_row.internal_id; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/down.sql b/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/down.sql new file mode 100644 index 00000000..6759492c --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/down.sql @@ -0,0 +1,2 @@ +ALTER TABLE ONLY node_transaction_history + DROP CONSTRAINT node_transaction_history_pkey; diff --git a/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/up.sql b/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/up.sql new file mode 100644 index 00000000..84283256 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778435997270_add_node_transaction_history_primary_key/up.sql @@ -0,0 +1,2 @@ +ALTER TABLE ONLY node_transaction_history + ADD CONSTRAINT node_transaction_history_pkey PRIMARY KEY (internal_id); diff --git a/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/down.sql b/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/down.sql new file mode 100644 index 00000000..6e599caa --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/down.sql @@ -0,0 +1,52 @@ +CREATE OR REPLACE FUNCTION parse_bytecode_pattern_with_pushdata_lengths(bytecode bytea) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + pattern bytea := '\x'::bytea; + selected_byte integer; + scratch bytea; + i integer := 0; + bytecode_length integer := octet_length(bytecode); +BEGIN + WHILE i < bytecode_length LOOP + selected_byte := get_byte(bytecode, i); + pattern := pattern || substring(bytecode from (i + 1) for 1); + IF selected_byte > 78 OR selected_byte = 0 THEN + -- OP_0 (0) and all opcodes after OP_PUSHDATA_4 (78) are single-byte instructions + i := i + 1; + ELSIF selected_byte > 0 AND selected_byte <= 75 THEN + -- OP_PUSHBYTES_1 (1) through OP_PUSHBYTES_75 (75) directly indicate the length of pushed data + i := i + 1 + selected_byte; + ELSIF selected_byte = 76 THEN + IF bytecode_length - i < 3 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_1 reads one length-byte + pattern := pattern || substring(bytecode from (i + 2) for 1); -- append length byte + i := i + 2 + get_byte(bytecode, (i + 1)); + ELSIF selected_byte = 77 THEN + IF bytecode_length - i < 4 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_2 reads two length-bytes + scratch := substring(bytecode from (i + 2) for 2); + pattern := pattern || scratch; -- append length bytes + -- parse scratch as unsigned, two byte, little-endian number: + i := i + 3 + ((get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + ELSIF selected_byte = 78 THEN + IF bytecode_length - i < 6 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_4 reads four length-bytes + scratch := substring(bytecode from (i + 2) for 4); + pattern := pattern || scratch; -- append length bytes + -- parse scratch as unsigned, four byte, little-endian number: + i := i + 5 + ((get_byte(scratch, 3) << 24) | (get_byte(scratch, 2) << 16) | (get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + END IF; + END LOOP; + RETURN pattern; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/up.sql b/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/up.sql new file mode 100644 index 00000000..34b48764 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778437612917_fix_zero_length_pushdata_patterns/up.sql @@ -0,0 +1,52 @@ +CREATE OR REPLACE FUNCTION parse_bytecode_pattern_with_pushdata_lengths(bytecode bytea) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + pattern bytea := '\x'::bytea; + selected_byte integer; + scratch bytea; + i integer := 0; + bytecode_length integer := octet_length(bytecode); +BEGIN + WHILE i < bytecode_length LOOP + selected_byte := get_byte(bytecode, i); + pattern := pattern || substring(bytecode from (i + 1) for 1); + IF selected_byte > 78 OR selected_byte = 0 THEN + -- OP_0 (0) and all opcodes after OP_PUSHDATA_4 (78) are single-byte instructions + i := i + 1; + ELSIF selected_byte > 0 AND selected_byte <= 75 THEN + -- OP_PUSHBYTES_1 (1) through OP_PUSHBYTES_75 (75) directly indicate the length of pushed data + i := i + 1 + selected_byte; + ELSIF selected_byte = 76 THEN + IF bytecode_length - i < 2 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_1 reads one length-byte + pattern := pattern || substring(bytecode from (i + 2) for 1); -- append length byte + i := i + 2 + get_byte(bytecode, (i + 1)); + ELSIF selected_byte = 77 THEN + IF bytecode_length - i < 3 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_2 reads two length-bytes + scratch := substring(bytecode from (i + 2) for 2); + pattern := pattern || scratch; -- append length bytes + -- parse scratch as unsigned, two byte, little-endian number: + i := i + 3 + ((get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + ELSIF selected_byte = 78 THEN + IF bytecode_length - i < 5 THEN + -- malformed, return immediately + RETURN pattern; + END IF; + -- OP_PUSHDATA_4 reads four length-bytes + scratch := substring(bytecode from (i + 2) for 4); + pattern := pattern || scratch; -- append length bytes + -- parse scratch as unsigned, four byte, little-endian number: + i := i + 5 + ((get_byte(scratch, 3) << 24) | (get_byte(scratch, 2) << 16) | (get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + END IF; + END LOOP; + RETURN pattern; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/down.sql b/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/down.sql new file mode 100644 index 00000000..beea7548 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/down.sql @@ -0,0 +1,49 @@ +CREATE OR REPLACE FUNCTION parse_bytecode_pattern_redeem(bytecode bytea) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + maybe_redeem bytea; + selected_byte integer; + length_value integer; + scratch bytea; + i integer := 0; + bytecode_length integer := octet_length(bytecode); +BEGIN + WHILE i < bytecode_length LOOP + selected_byte := get_byte(bytecode, i); + IF selected_byte > 78 OR selected_byte = 0 THEN + -- OP_0 (0) and all opcodes after OP_PUSHDATA_4 (78) are single-byte instructions + i := i + 1; + maybe_redeem := NULL; + ELSIF selected_byte > 0 AND selected_byte <= 75 THEN + -- OP_PUSHBYTES_1 (1) through OP_PUSHBYTES_75 (75) directly indicate the length of pushed data + maybe_redeem := substring(bytecode from (i + 2) for selected_byte); + i := i + 1 + selected_byte; + ELSIF selected_byte = 76 THEN + -- OP_PUSHDATA_1 reads one length-byte + length_value := get_byte(bytecode, (i + 1)); + maybe_redeem := substring(bytecode from (i + 3) for length_value); + i := i + 2 + length_value; + ELSIF selected_byte = 77 THEN + -- OP_PUSHDATA_2 reads two length-bytes + scratch := substring(bytecode from (i + 2) for 2); + -- parse scratch as unsigned, two byte, little-endian number: + length_value := ((get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + maybe_redeem := substring(bytecode from (i + 4) for length_value); + i := i + 3 + length_value; + ELSIF selected_byte = 78 THEN + -- OP_PUSHDATA_4 reads four length-bytes + scratch := substring(bytecode from (i + 2) for 4); + -- parse scratch as unsigned, four byte, little-endian number: + length_value := ((get_byte(scratch, 3) << 24) | (get_byte(scratch, 2) << 16) | (get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + maybe_redeem := substring(bytecode from (i + 6) for length_value); + i := i + 5 + length_value; + END IF; + END LOOP; + IF maybe_redeem = NULL THEN + RETURN maybe_redeem; + ELSE + RETURN parse_bytecode_pattern(maybe_redeem); + END IF; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/up.sql b/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/up.sql new file mode 100644 index 00000000..982c46b5 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1778438318512_fix_redeem_bytecode_parser/up.sql @@ -0,0 +1,81 @@ +CREATE OR REPLACE FUNCTION parse_bytecode_pattern_redeem(bytecode bytea) RETURNS bytea + LANGUAGE plpgsql IMMUTABLE +AS $$ +DECLARE + maybe_redeem bytea; + selected_byte integer; + length_value integer; + scratch bytea; + i integer := 0; + bytecode_length integer := octet_length(bytecode); +BEGIN + WHILE i < bytecode_length LOOP + selected_byte := get_byte(bytecode, i); + IF selected_byte > 78 OR selected_byte = 0 THEN + -- OP_0 (0) and all opcodes after OP_PUSHDATA_4 (78) are single-byte instructions + i := i + 1; + maybe_redeem := NULL; + ELSIF selected_byte > 0 AND selected_byte <= 75 THEN + -- OP_PUSHBYTES_1 (1) through OP_PUSHBYTES_75 (75) directly indicate the length of pushed data + IF bytecode_length - i < 1 + selected_byte THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + maybe_redeem := substring(bytecode from (i + 2) for selected_byte); + i := i + 1 + selected_byte; + ELSIF selected_byte = 76 THEN + IF bytecode_length - i < 2 THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + -- OP_PUSHDATA_1 reads one length-byte + length_value := get_byte(bytecode, (i + 1)); + IF bytecode_length - i < 2 + length_value THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + maybe_redeem := substring(bytecode from (i + 3) for length_value); + i := i + 2 + length_value; + ELSIF selected_byte = 77 THEN + IF bytecode_length - i < 3 THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + -- OP_PUSHDATA_2 reads two length-bytes + scratch := substring(bytecode from (i + 2) for 2); + -- parse scratch as unsigned, two byte, little-endian number: + length_value := ((get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + IF bytecode_length - i < 3 + length_value THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + maybe_redeem := substring(bytecode from (i + 4) for length_value); + i := i + 3 + length_value; + ELSIF selected_byte = 78 THEN + IF bytecode_length - i < 5 THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + -- OP_PUSHDATA_4 reads four length-bytes + scratch := substring(bytecode from (i + 2) for 4); + IF get_byte(scratch, 3) << 24 < 0 THEN + -- push length exceeds maximum signed int (>2GB) + RETURN NULL; + END IF; + -- parse scratch as unsigned, four byte, little-endian number: + length_value := ((get_byte(scratch, 3) << 24) | (get_byte(scratch, 2) << 16) | (get_byte(scratch, 1) << 8) | get_byte(scratch, 0)); + IF bytecode_length - i - 5 < length_value THEN + -- malformed, no redeem bytecode can be safely identified + RETURN NULL; + END IF; + maybe_redeem := substring(bytecode from (i + 6) for length_value); + i := i + 5 + length_value; + END IF; + END LOOP; + IF maybe_redeem IS NULL THEN + RETURN maybe_redeem; + ELSE + RETURN parse_bytecode_pattern(maybe_redeem); + END IF; +END; +$$; diff --git a/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/down.sql b/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/down.sql new file mode 100644 index 00000000..53c12d2d --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/down.sql @@ -0,0 +1,17 @@ +-- Restore the original search_output definition from 1616195337538_init. +CREATE OR REPLACE FUNCTION search_output (locking_bytecode_hex text[]) + RETURNS SETOF output + LANGUAGE plpgsql STABLE +AS $$ +DECLARE + decoded bytea[]; +BEGIN + SELECT array_agg(decode(hex, 'hex')) into decoded FROM unnest(locking_bytecode_hex) as hex; + RETURN QUERY (SELECT * FROM output + -- use output_locking_bytecode_prefix_index + WHERE substring(locking_bytecode from 0 for 26) + = ANY(decoded) + ORDER BY locking_bytecode ASC); +END; +$$; +COMMENT ON FUNCTION search_output (text[]) IS 'Return a list of outputs which match the provided locking bytecode hex (up to 25 bytes, supporting both P2PKH and P2SH outputs).'; diff --git a/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/up.sql b/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/up.sql new file mode 100644 index 00000000..97358cc0 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791100000000_fix_search_output_exact_matching/up.sql @@ -0,0 +1,20 @@ +-- search_output previously compared each input, unchanged, with the 25-byte +-- prefix stored by output_search_index, so inputs longer than 25 bytes (e.g. +-- P2SH32) never matched and 25-byte inputs also matched longer scripts sharing +-- that prefix. Truncate each input to 25 bytes for the indexed comparison, then +-- recheck full equality. As a single-statement SQL function, PostgreSQL can +-- inline it, so Hasura's where/limit/order_by apply inside the query. +-- Note: this method expects an index: CREATE INDEX output_search_index ON output USING btree (substring(locking_bytecode, 0, 26)); +CREATE OR REPLACE FUNCTION search_output (locking_bytecode_hex text[]) + RETURNS SETOF output + LANGUAGE sql STABLE +AS $$ + SELECT * FROM output + -- use output_search_index (first 25 bytes), then compare the full bytecode + WHERE substring(locking_bytecode from 0 for 26) = ANY (ARRAY( + SELECT substring(decode(hex, 'hex') from 1 for 25) FROM unnest(locking_bytecode_hex) AS hex)) + AND locking_bytecode = ANY (ARRAY( + SELECT decode(hex, 'hex') FROM unnest(locking_bytecode_hex) AS hex)) + ORDER BY locking_bytecode ASC +$$; +COMMENT ON FUNCTION search_output (text[]) IS 'Return a list of outputs whose locking bytecode exactly matches any of the provided locking bytecode hex values (of any length, e.g. P2PKH, P2SH20 and P2SH32).'; diff --git a/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/down.sql b/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/down.sql new file mode 100644 index 00000000..0cb663be --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/down.sql @@ -0,0 +1,12 @@ +-- Restore the original search_output_prefix definition from 1616195337538_init. +CREATE OR REPLACE FUNCTION search_output_prefix (locking_bytecode_prefix_hex text) + RETURNS SETOF output + LANGUAGE sql STABLE +AS $$ + SELECT * FROM output + -- use output_locking_bytecode_prefix_index + WHERE substring(locking_bytecode from 0 for 26) + LIKE decode(locking_bytecode_prefix_hex, 'hex') || '%' + ORDER BY locking_bytecode ASC +$$; +COMMENT ON FUNCTION search_output_prefix (text) IS 'Return a list of outputs in which the first 25 bytes of the locking bytecode match the provided prefix hex.'; diff --git a/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql b/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql new file mode 100644 index 00000000..143de68d --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql @@ -0,0 +1,22 @@ +-- search_output_prefix previously matched with LIKE, so bytes 0x5c (escape), +-- 0x25 (%) and 0x5f (_) in the decoded prefix were treated as pattern syntax, +-- and prefixes longer than 25 bytes never matched. Select a byte range on the +-- 25-byte expression indexed by output_search_index, then recheck the exact +-- prefix against the full locking bytecode. +-- Note: this method expects an index: CREATE INDEX output_search_index ON output USING btree (substring(locking_bytecode, 0, 26)); +CREATE OR REPLACE FUNCTION search_output_prefix (locking_bytecode_prefix_hex text) + RETURNS SETOF output + LANGUAGE sql STABLE +AS $$ + SELECT * FROM output + -- use output_search_index: every 25-byte value starting with the prefix lies + -- between the prefix and the prefix padded with 0xff bytes to 25 bytes + WHERE substring(locking_bytecode from 0 for 26) + BETWEEN substring(decode(locking_bytecode_prefix_hex, 'hex') from 1 for 25) + AND substring(decode(locking_bytecode_prefix_hex, 'hex') from 1 for 25) + || decode(repeat('ff', 25 - least(length(decode(locking_bytecode_prefix_hex, 'hex')), 25)), 'hex') + AND substring(locking_bytecode from 1 for length(decode(locking_bytecode_prefix_hex, 'hex'))) + = decode(locking_bytecode_prefix_hex, 'hex') + ORDER BY locking_bytecode ASC +$$; +COMMENT ON FUNCTION search_output_prefix (text) IS 'Return a list of outputs whose locking bytecode begins with the provided prefix hex (of any length; every byte is matched literally).'; diff --git a/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/down.sql b/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/down.sql new file mode 100644 index 00000000..0aaf0256 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS incomplete_block_audit; diff --git a/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/up.sql b/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/up.sql new file mode 100644 index 00000000..381894e5 --- /dev/null +++ b/images/hasura/hasura-data/migrations/default/1791200000000_add_incomplete_block_audit/up.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS incomplete_block_audit ( + node_internal_id integer NOT NULL, + audited_through_block_internal_id bigint NOT NULL, + completed_at timestamp without time zone DEFAULT now() NOT NULL, + CONSTRAINT incomplete_block_audit_pkey PRIMARY KEY (node_internal_id), + CONSTRAINT incomplete_block_audit_node_internal_id_fkey FOREIGN KEY (node_internal_id) REFERENCES node(internal_id) ON UPDATE RESTRICT ON DELETE CASCADE +); +COMMENT ON TABLE incomplete_block_audit IS 'Internal agent bookkeeping for the incomplete block repair scan (https://github.com/bitauth/chaingraph/issues/74). Each row records that every block accepted by the node with internal_id at or below audited_through_block_internal_id has been audited, so the agent skips those blocks on later startups. Delete rows to force a full re-audit.'; +COMMENT ON COLUMN incomplete_block_audit.node_internal_id IS 'The internal_id of the node whose accepted blocks were audited.'; +COMMENT ON COLUMN incomplete_block_audit.audited_through_block_internal_id IS 'The highest block internal_id which existed when the completed audit began.'; +COMMENT ON COLUMN incomplete_block_audit.completed_at IS 'The time at which the audit completed.'; diff --git a/package.json b/package.json index c93c11cb..9c81f48f 100644 --- a/package.json +++ b/package.json @@ -142,14 +142,14 @@ "postinstall": "touch .env" }, "engines": { - "node": ">=10" + "node": ">=22" }, "dependencies": { "@bitauth/libauth": "^2.0.0-alpha.8", "@chaingraph/bitcore-p2p-cash": "^9.0.5", "dotenv": "^16.0.3", "lru-cache": "^7.14.1", - "pg": "^8.8.0", + "pg": "^8.22.0", "pino": "^8.8.0" }, "devDependencies": { @@ -157,7 +157,7 @@ "@istanbuljs/nyc-config-typescript": "^1.0.2", "@types/dotenv": "^8.2.0", "@types/lru-cache": "^7.10.10", - "@types/pg": "^8.6.6", + "@types/pg": "^8.20.0", "@typescript-eslint/eslint-plugin": "^5.48.1", "@typescript-eslint/parser": "^5.48.1", "ava": "^5.1.0", diff --git a/src/agent.ts b/src/agent.ts index 3241571a..9d8d6f1a 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -16,10 +16,12 @@ import type pino from 'pino'; import { bitcoreBlockToChaingraphBlock, bitcoreTransactionToChaingraphTransaction, + createPeerMessages, } from './bitcore.js'; import { BlockBuffer } from './components/block-buffer.js'; import { BlockTree } from './components/block-tree.js'; import type { indexDefinitions } from './components/db-utils.js'; +import { MempoolResync } from './components/mempool-resync.js'; import { SyncState } from './components/sync-state.js'; import { formatBytes, @@ -31,14 +33,24 @@ import { chaingraphLogFirehose, chaingraphUserAgent, genesisBlocks, + incompleteBlockRepairBatchSize, + mempoolResync, + mempoolResyncMinIntervalMs, + mempoolTransactionExpirationMs, + mempoolTransactionExpirationScanIntervalMs, postgresMaxConnections, trustedNodes, } from './config.js'; import { acceptBlocksViaHeaders, + archiveMempoolTransactionsAcceptedByBlocks, createIndexes, + expireMempoolTransaction, getAllKnownBlockHashes, + getIncompleteBlockAuditState, + getIncompleteBlocks, getIndexCreationProgress, + getMempoolTransactionsExpiringBefore, listExistingIndexes, optionallyDisableSynchronousCommit, optionallyEnableSynchronousCommit, @@ -48,8 +60,18 @@ import { registerTrustedNodeWithDb, removeStaleBlocksForNode, saveBlock, + saveIncompleteBlockAudit, saveTransactionForNodes, } from './db.js'; +import type { + ExpiringMempoolTransaction, + IncompleteBlock, + IncompleteBlockAuditState, +} from './db.js'; +import { + describePostgresError, + isPostgresConnectionError, +} from './postgres-errors.js'; import type { ChaingraphBlock } from './types/chaingraph.js'; // eslint-disable-next-line @typescript-eslint/naming-convention @@ -108,6 +130,32 @@ const renderSyncPercentage = (value: number) => { }; const msPerSecond = 1000; +/** + * Delay before retrying an incomplete block repair scan which failed because + * the Postgres connection was lost (e.g. Postgres is restarting). + */ +const incompleteBlockRepairRetryDelayMs = 5_000; +const durationDecimalPlaces = 1; +const transactionRateDecimalPlaces = 2; + +const formatDurationSeconds = (durationMs: number) => + `${(durationMs / msPerSecond).toFixed(durationDecimalPlaces)}s`; + +const formatTransactionRate = (transactionCount: number, durationMs: number) => + durationMs === 0 + ? 'n/a tx/s' + : `${((transactionCount * msPerSecond) / durationMs).toFixed( + transactionRateDecimalPlaces + )}tx/s`; + +const formatTransactionDuration = ( + transactionCount: number, + durationMs: number +) => + transactionCount === 0 + ? 'n/a/tx' + : `${formatDurationSeconds(durationMs / transactionCount)}/tx`; + /** * Convert a bitcoin block header timestamp (UTC in seconds) to a `Date`. * @param timestamp - the block header timestamp @@ -284,10 +332,76 @@ export class Agent { nodesByInternalId = [] as [nodeName: string, node: Node][]; + /** + * Requests the mempool of trusted nodes (BIP35) after mempool tracking is + * enabled, on reconnection, and after re-organizations, so transactions + * which nodes don't re-announce (e.g. those returned to the mempool by a + * re-organization) are recorded in `node_transaction`. See + * https://github.com/bitauth/chaingraph/issues/81. + */ + mempoolResync = new MempoolResync({ + enabled: mempoolResync, + getReadyNode: (nodeName) => { + const node = this.nodes[nodeName]; + return node?.internalId !== undefined && node.peer.status === 'ready' + ? node + : undefined; + }, + minimumIntervalMs: mempoolResyncMinIntervalMs, + onRequestSent: (nodeName, triggers) => { + this.logger.info( + `${nodeName}: requested node mempool (BIP35) – triggers: ${triggers.join( + ', ' + )}` + ); + }, + transactionCache: this.transactionCache, + }); + heartbeatInterval: NodeJS.Timeout; scheduledBlockBufferFill = false; + scheduledIncompleteBlockRepair = false; + + incompleteBlockRepairInProgress = false; + + currentIncompleteBlockRepair: + | { + hash: string; + resolve: () => void; + } + | undefined; + + incompleteBlockRepairTimeout: ReturnType | undefined; + + mempoolTransactionExpirationScanTimeout: + | ReturnType + | undefined; + + mempoolTransactionExpirationTimers = new Map< + string, + ReturnType + >(); + + incompleteBlockRepairNextHeight = 0; + + completedIncompleteBlockRepairScan = false; + + /** + * State of the in-progress incomplete block repair scan: the audit markers + * loaded when it began, and the nodes which were registered for every batch + * (only those nodes' markers are advanced when the scan completes). + */ + incompleteBlockAuditPass: + | { + auditState: IncompleteBlockAuditState; + auditedBlockCount: number; + nodeInternalIdsInEveryBatch: number[]; + startedAt: number; + } + | undefined; + /** * The next second after which to log another warning that one or more nodes * are unresponsive. @@ -326,6 +440,7 @@ export class Agent { 'block_inclusions_index', 'output_search_index', 'spent_by_index', + 'token_category_index', ]; /** @@ -370,10 +485,19 @@ export class Agent { targetSize: blockBufferTargetSizeMb * MB, }); + const blockDbRestoreStart = Date.now(); // eslint-disable-next-line @typescript-eslint/no-floating-promises getAllKnownBlockHashes().then((hashes) => { this.blockDb = new Set(hashes); this.blockDbRestored = true; + const millisecondsPerSecond = 1000; + const restoreSeconds = ( + (Date.now() - blockDbRestoreStart) / + millisecondsPerSecond + ).toFixed(1); + this.logger.info( + `Restored ${hashes.length} known block hashes from the database in ${restoreSeconds} seconds.` + ); }); this.logger.info( @@ -431,7 +555,10 @@ export class Agent { subversion: chaingraphUserAgent, version: chaingraphProtocolVersion, }; - const peer = new Peer(peerConfig); + const peer = new Peer({ + ...peerConfig, + messages: createPeerMessages(node.networkMagicHex), + }); const peerTxBroadcastConnection = new Peer({ ...peerConfig, subversion: `${chaingraphUserAgent}tx-broadcast/`, @@ -481,6 +608,7 @@ export class Agent { */ peer.sendMessage(new peer.messages.SendHeaders()); + const nodeRegistrationStart = Date.now(); // eslint-disable-next-line @typescript-eslint/no-floating-promises registerTrustedNodeWithDb({ latestConnectionBeganAt: new Date(), @@ -503,6 +631,14 @@ export class Agent { : syncedHeaderHashChain ); this.nodes[node.name]!.syncState = new SyncState(initialSyncState); + const millisecondsPerSecond = 1000; + const chainRestoreSeconds = ( + (Date.now() - nodeRegistrationStart) / + millisecondsPerSecond + ).toFixed(1); + this.logger.info( + `${node.name}: chain state registered and restored from the database in ${chainRestoreSeconds} seconds.` + ); const genesisBlockHeaderFromDb = this.blockTree.getBlockHeaderHash( node.name, 0 @@ -529,6 +665,7 @@ export class Agent { } nodeRegisteredResolver(); this.requestHeaders(node.name); + this.mempoolResync.handleConnect(node.name); }); }); @@ -709,6 +846,47 @@ export class Agent { * issue requires that Chaingraph be restarted. */ + warnIfInitializationDelayed(uninitializedNodes: string[]) { + const second = 1000; + const currentDelay = Date.now() - this.startTime.getTime(); + const currentDelaySecondsRounded = Math.round(currentDelay / second); + if (this.warnAfterDelaySecond >= currentDelaySecondsRounded) { + return; + } + this.warnAfterDelaySecond = currentDelaySecondsRounded; + if (!this.blockDbRestored) { + this.logger.warn( + `Still restoring known block hashes from the database (${currentDelaySecondsRounded} seconds elapsed). This can take a minute or more on large databases; if it never completes, check database configuration and connectivity.` + ); + return; + } + /* + * A node whose P2P connection is `ready` is not unresponsive – it is + * waiting on its chain state to be registered and restored from the + * database, which takes time proportional to the node's chain length. + */ + const unconnectedNodes = uninitializedNodes.filter( + (id) => this.nodes[id]!.peer.status !== 'ready' + ); + const restoringChainNodes = uninitializedNodes.filter( + (id) => this.nodes[id]!.peer.status === 'ready' + ); + if (unconnectedNodes.length > 0) { + this.logger.warn( + `The following nodes are taking longer than ${currentDelaySecondsRounded} seconds to connect and may be either misconfigured or unresponsive: ${unconnectedNodes.join( + ',' + )}` + ); + } + if (restoringChainNodes.length > 0) { + this.logger.warn( + `Still restoring chain state from the database for: ${restoringChainNodes.join( + ',' + )} (${currentDelaySecondsRounded} seconds elapsed). Restore time is proportional to each node's chain length.` + ); + } + } + checkInitialization() { const uninitializedNodes = Object.entries(this.nodes).reduce( (nodes, [id, node]) => @@ -717,22 +895,7 @@ export class Agent { ); if (uninitializedNodes.length > 0 || !this.blockDbRestored) { const second = 1000; - const currentDelay = Date.now() - this.startTime.getTime(); - const currentDelaySecondsRounded = Math.round(currentDelay / second); - if (this.warnAfterDelaySecond < currentDelaySecondsRounded) { - this.warnAfterDelaySecond = currentDelaySecondsRounded; - if (this.blockDbRestored) { - this.logger.warn( - `The following nodes are taking longer than ${currentDelaySecondsRounded} seconds to initialize and may be either misconfigured or unresponsive: ${uninitializedNodes.join( - ',' - )}` - ); - } else { - this.logger.warn( - `Could not restore block hashes from the database after ${currentDelaySecondsRounded} seconds. There may be a database configuration or connectivity problem.` - ); - } - } + this.warnIfInitializationDelayed(uninitializedNodes); setTimeout(() => { this.scheduleBlockBufferFill(); }, second); @@ -825,9 +988,19 @@ export class Agent { this.logger.info( `Agent: all managed indexes have been created.` ); - return reenableMempoolCleaning().then(() => { + return reenableMempoolCleaning().then((schemaIsCurrent) => { + if (!schemaIsCurrent) { + this.logger.warn( + 'Agent: WARNING! Database schema is old and missing multiple bug fixes. Update the Hasura image to apply migrations and then restart this agent.' + ); + } this.logger.info('Agent: enabled mempool tracking.'); this.saveInboundTransactions = true; + this.mempoolResync.enableMempoolTracking( + Object.keys(this.nodes) + ); + this.scheduleIncompleteBlockRepair(); + this.scheduleMempoolTransactionExpirationScan(); }); }) .catch((err) => { @@ -906,6 +1079,436 @@ export class Agent { return indexCreationCompletion; } + canScanForMempoolTransactionExpirations() { + return ![ + !this.completedInitialSync, + !this.saveInboundTransactions, + this.waitingForIncompleteBlockRepairScan(), + this.mempoolTransactionExpirationScanTimeout !== undefined, + this.willShutdown, + ].includes(true); + } + + waitingForIncompleteBlockRepairScan() { + return ( + incompleteBlockRepairBatchSize !== 0 && + !this.completedIncompleteBlockRepairScan + ); + } + + scheduleMempoolTransactionExpirationScan(delayMs = 0, forceSchedule = false) { + if ( + (!forceSchedule && !this.canScanForMempoolTransactionExpirations()) || + this.willShutdown + ) { + return; + } + this.mempoolTransactionExpirationScanTimeout = setTimeout(() => { + this.mempoolTransactionExpirationScanTimeout = undefined; + this.scanForMempoolTransactionExpirations() + .catch((err) => { + this.logger.error( + err, + 'Agent: failed to scan for expiring mempool transactions.' + ); + }) + .finally(() => { + this.scheduleMempoolTransactionExpirationScan( + mempoolTransactionExpirationScanIntervalMs, + true + ); + }); + }, delayMs); + } + + scheduleMempoolTransactionExpiration( + transaction: ExpiringMempoolTransaction + ) { + const key = `${transaction.nodeInternalId}:${transaction.transactionInternalId}`; + if (this.mempoolTransactionExpirationTimers.has(key)) { + return; + } + const delayMs = Math.max(transaction.expiresAt.getTime() - Date.now(), 0); + const timeout = setTimeout(() => { + this.mempoolTransactionExpirationTimers.delete(key); + this.expireMempoolTransaction(transaction).catch((err) => { + this.logger.error( + err, + `Agent: failed to expire mempool transaction ${transaction.hash} for node ${transaction.nodeName}.` + ); + }); + }, delayMs); + this.mempoolTransactionExpirationTimers.set(key, timeout); + } + + async scanForMempoolTransactionExpirations() { + if (!this.saveInboundTransactions || this.willShutdown) { + return; + } + await this.archiveAcceptedMempoolTransactions(); + const expiresBefore = new Date( + Date.now() + mempoolTransactionExpirationScanIntervalMs + ); + const expiringTransactions = await getMempoolTransactionsExpiringBefore({ + expirationMs: mempoolTransactionExpirationMs, + expiresBefore, + }); + if (expiringTransactions.length === 0) { + this.logger.debug( + `Agent: no mempool transactions expiring before ${expiresBefore.toISOString()}; next scan in ${mempoolTransactionExpirationScanIntervalMs.toLocaleString()}ms.` + ); + return; + } + this.logger.info( + `Agent: found ${ + expiringTransactions.length + } mempool transaction(s) expiring before ${expiresBefore.toISOString()}; scheduling exact expiry.` + ); + expiringTransactions.forEach((transaction) => { + this.scheduleMempoolTransactionExpiration(transaction); + }); + } + + async expireMempoolTransaction(transaction: ExpiringMempoolTransaction) { + // queued: expiries run one ordered batch at a time (see expireMempoolTransaction in db.ts) + const archived = await expireMempoolTransaction({ + nodeInternalId: transaction.nodeInternalId, + replacedAt: transaction.expiresAt, + transactionInternalId: transaction.transactionInternalId, + validatedAt: transaction.validatedAt, + }); + if (!archived) { + this.logger.debug( + `Agent: skipped expiry of mempool transaction ${transaction.hash} for node ${transaction.nodeName}; it has already left node_transaction, or it (or a conflicting spend) is in a block accepted by the node and is left for the accepted-block archive.` + ); + return; + } + this.logger.warn( + `Agent: expired mempool transaction ${transaction.hash} for node ${ + transaction.nodeName + }; archived from node_transaction to node_transaction_history with replaced_at ${transaction.expiresAt.toISOString()}.` + ); + } + + async archiveAcceptedMempoolTransactions() { + const archivedTransactions = + await archiveMempoolTransactionsAcceptedByBlocks(); + if (archivedTransactions.length === 0) { + return; + } + this.logger.warn( + `Agent: archived ${archivedTransactions.length.toLocaleString()} stale mempool transaction(s) already accepted or replaced by blocks before ordinary expiry.` + ); + archivedTransactions.forEach((transaction) => { + if (transaction.replacedAt === null) { + this.logger.info( + `Agent: archived stale confirmed mempool transaction ${transaction.hash} for node ${transaction.nodeName}; transaction is already accepted by a block, archived with replaced_at NULL.` + ); + return; + } + this.logger.info( + `Agent: archived stale replaced mempool transaction ${ + transaction.hash + } for node ${ + transaction.nodeName + }; an accepted block already spends the same outpoint, archived with replaced_at ${transaction.replacedAt.toISOString()}.` + ); + }); + } + + canScheduleIncompleteBlockRepair() { + return ![ + incompleteBlockRepairBatchSize === 0, + !this.completedInitialSync, + this.scheduledIncompleteBlockRepair, + this.incompleteBlockRepairInProgress, + this.completedIncompleteBlockRepairScan, + this.willShutdown, + ].includes(true); + } + + scheduleIncompleteBlockRepair() { + if (!this.canScheduleIncompleteBlockRepair()) { + return; + } + this.incompleteBlockRepairTimeout = setTimeout(() => { + this.scheduledIncompleteBlockRepair = false; + this.repairIncompleteBlocks().catch((err) => { + if (isPostgresConnectionError(err) && !this.willShutdown) { + this.logger.warn( + `Agent: incomplete block repair scan lost its Postgres connection (${describePostgresError( + err + )}); retrying in ${ + incompleteBlockRepairRetryDelayMs / msPerSecond + } seconds.` + ); + this.incompleteBlockRepairTimeout = setTimeout(() => { + this.incompleteBlockRepairTimeout = undefined; + this.scheduleIncompleteBlockRepair(); + }, incompleteBlockRepairRetryDelayMs); + return; + } + this.logger.fatal(err); + this.shutdown().catch((shutdownErr) => { + this.logger.error(shutdownErr); + }); + }); + }); + this.scheduledIncompleteBlockRepair = true; + } + + getIncompleteBlockRepairRange() { + const bestHeight = Math.max( + ...Object.values(this.blockTree.getBestHeights()) + ); + const finalHeight = bestHeight + 1; + const heightLowerBound = + this.incompleteBlockRepairNextHeight > bestHeight + ? 0 + : this.incompleteBlockRepairNextHeight; + const heightUpperBound = Math.min( + heightLowerBound + incompleteBlockRepairBatchSize, + finalHeight + ); + return { finalHeight, heightLowerBound, heightUpperBound }; + } + + updateIncompleteBlockRepairProgress({ + finalHeight, + heightLowerBound, + heightUpperBound, + incompleteBlockCount, + limit, + }: { + finalHeight: number; + heightLowerBound: number; + heightUpperBound: number; + incompleteBlockCount: number; + limit: number; + }) { + if (incompleteBlockCount === limit) { + this.incompleteBlockRepairNextHeight = heightLowerBound; + return; + } + if (heightUpperBound === finalHeight) { + this.incompleteBlockRepairNextHeight = 0; + this.completedIncompleteBlockRepairScan = true; + this.logger.info('Agent: completed incomplete block repair scan.'); + this.scheduleMempoolTransactionExpirationScan(); + return; + } + this.incompleteBlockRepairNextHeight = heightUpperBound; + } + + async repairIncompleteBlock(block: IncompleteBlock) { + if (this.currentIncompleteBlockRepair !== undefined) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Agent: attempted to repair incomplete block ${block.height} (${block.hash}) while already repairing ${this.currentIncompleteBlockRepair.hash}.` + ); + } + const sourceNodes = this.blockTree.getNodesWithBlock( + block.hash, + block.height + ); + if (sourceNodes.length === 0) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Agent: incomplete block ${block.height} (${block.hash}) is not currently accepted by any connected node.` + ); + } + this.logger.info( + `Agent: self-healing incomplete block ${block.height} (${block.hash}); linked size ${block.linkedSizeBytes}/${block.sizeBytes} bytes across ${block.transactionCount} saved transaction(s).` + ); + const alreadyDownloading = this.blockDownloads.some( + (download) => download.hash === block.hash + ); + return new Promise((resolve) => { + this.currentIncompleteBlockRepair = { + hash: block.hash, + resolve, + }; + if (!alreadyDownloading) { + this.blockBuffer.reserveBlock(); + this.requestBlock(block.hash, block.height); + } + }); + } + + canRepairIncompleteBlocks() { + return ( + incompleteBlockRepairBatchSize !== 0 && + this.completedInitialSync && + !this.willShutdown + ); + } + + getRegisteredNodeInternalIds() { + return Object.values(this.nodes) + .map((node) => node.internalId) + .filter((id): id is number => id !== undefined); + } + + async repairIncompleteBlocksSequentially(blocks: IncompleteBlock[]) { + await blocks.reduce>(async (previousRepair, block) => { + await previousRepair; + await this.repairIncompleteBlock(block); + }, Promise.resolve()); + } + + /** + * Begin (or continue) an incomplete block audit pass. Blocks which a + * previous, completed pass already audited for a node are skipped for that + * node, so restarting a synced agent doesn't re-read every saved + * transaction (~51 GB on an 831k-block mainnet database). + */ + async getIncompleteBlockAuditPass(nodeInternalIds: number[]) { + if (this.incompleteBlockAuditPass !== undefined) { + return this.incompleteBlockAuditPass; + } + const auditState = await getIncompleteBlockAuditState(nodeInternalIds); + const auditPass = { + auditState, + auditedBlockCount: 0, + nodeInternalIdsInEveryBatch: nodeInternalIds, + startedAt: Date.now(), + }; + this.incompleteBlockAuditPass = auditPass; + if (!auditState.auditTableExists) { + this.logger.warn( + 'Agent: database schema has no incomplete_block_audit table, so the incomplete block repair scan will audit every saved block on each startup. Update the Hasura image to apply migrations.' + ); + return auditPass; + } + const markers = nodeInternalIds.map( + (id) => + `${id}: ${auditState.auditedThroughBlockInternalIds.get(id) ?? 'none'}` + ); + this.logger.info( + `Agent: starting incomplete block repair scan; skipping blocks already audited per node (node internal_id: audited through block internal_id) – ${markers.join( + ', ' + )}.` + ); + return auditPass; + } + + async completeIncompleteBlockAuditPass() { + const auditPass = this.incompleteBlockAuditPass; + this.incompleteBlockAuditPass = undefined; + if (auditPass === undefined) { + return; + } + const { auditState, auditedBlockCount, nodeInternalIdsInEveryBatch } = + auditPass; + const durationSeconds = ( + (Date.now() - auditPass.startedAt) / + msPerSecond + ).toFixed(1); + this.logger.info( + `Agent: incomplete block repair scan audited ${auditedBlockCount.toLocaleString()} block(s) in ${durationSeconds} seconds.` + ); + if (!auditState.auditTableExists) { + return; + } + await saveIncompleteBlockAudit({ + auditedThroughBlockInternalId: auditState.maxBlockInternalId, + nodeInternalIds: nodeInternalIdsInEveryBatch, + }) + .then(() => { + this.logger.debug( + `Agent: recorded incomplete block audit through block internal_id ${ + auditState.maxBlockInternalId + } for node internal_id(s): ${nodeInternalIdsInEveryBatch.join(', ')}.` + ); + }) + .catch((err) => { + this.logger.error( + err, + 'Agent: failed to record incomplete block audit; the next startup will audit these blocks again.' + ); + }); + } + + async repairIncompleteBlocksOnce() { + const { finalHeight, heightLowerBound, heightUpperBound } = + this.getIncompleteBlockRepairRange(); + const nodeInternalIds = this.getRegisteredNodeInternalIds(); + const auditPass = await this.getIncompleteBlockAuditPass(nodeInternalIds); + auditPass.nodeInternalIdsInEveryBatch = + auditPass.nodeInternalIdsInEveryBatch.filter((id) => + nodeInternalIds.includes(id) + ); + const scanStartTime = Date.now(); + const { incompleteBlocks, scannedBlockCount } = await getIncompleteBlocks({ + auditedThroughBlockInternalIds: nodeInternalIds.map( + (id) => auditPass.auditState.auditedThroughBlockInternalIds.get(id) ?? 0 + ), + excludedBlockHashes: [], + heightLowerBound, + heightUpperBound, + limit: incompleteBlockRepairBatchSize, + nodeInternalIds, + }); + const scanDurationMs = Date.now() - scanStartTime; + const scanRate = + scanDurationMs === 0 + ? scannedBlockCount * msPerSecond + : Math.round((scannedBlockCount / scanDurationMs) * msPerSecond); + const scanPerformanceLog = `scanned ${scannedBlockCount.toLocaleString()} block(s) in ${scanDurationMs.toLocaleString()}ms (${scanRate.toLocaleString()} blocks/s)`; + if (incompleteBlocks.length === 0) { + auditPass.auditedBlockCount += scannedBlockCount; + if (heightUpperBound === finalHeight) { + await this.completeIncompleteBlockAuditPass(); + } + this.updateIncompleteBlockRepairProgress({ + finalHeight, + heightLowerBound, + heightUpperBound, + incompleteBlockCount: incompleteBlocks.length, + limit: incompleteBlockRepairBatchSize, + }); + this.logger.info( + `Agent: no incomplete blocks found from height ${heightLowerBound} to ${ + heightUpperBound - 1 + }; ${scanPerformanceLog}.` + ); + return; + } + this.logger.warn( + `Agent: found ${ + incompleteBlocks.length + } incomplete block(s) from height ${heightLowerBound} to ${ + heightUpperBound - 1 + }; ${scanPerformanceLog}; requesting full block contents for repair.` + ); + await this.repairIncompleteBlocksSequentially(incompleteBlocks); + } + + /** + * Audit a bounded range of blocks accepted by currently-connected nodes. If + * the saved transactions don't sum to the saved block size, re-request the + * full block and let the normal block-saving path repair missing rows. + */ + async repairIncompleteBlocks() { + if (!this.canRepairIncompleteBlocks()) { + return; + } + if (this.incompleteBlockRepairInProgress) { + return; + } + this.incompleteBlockRepairInProgress = true; + await this.repairIncompleteBlocksOnce() + .then(() => { + this.incompleteBlockRepairInProgress = false; + this.scheduleIncompleteBlockRepair(); + }) + .catch((err) => { + this.incompleteBlockRepairInProgress = false; + // eslint-disable-next-line functional/no-throw-statement + throw err; + }); + } + /** * Internal method used to identify sync status of all nodes. Returns a list * of nodes in order from least-synced to most-synced. (Nodes which have not @@ -1362,10 +1965,12 @@ export class Agent { nodeAcceptances, transactionCache: this.transactionCache, }); + this.blockDb?.add(block.hash); const completionTime = Date.now(); const durationMs = completionTime - startTime; const transactions = attemptedSavedTransactions.length; + const savedTransactionCount = transactions - transactionCacheMisses; const inputs = attemptedSavedTransactions.reduce( (total, tx) => total + tx.inputs.length, 0 @@ -1399,15 +2004,29 @@ export class Agent { .toString() .padStart(heightMinWidth, ' ')} | timestamp: ${blockTimestampToDate( block.timestamp - ).toISOString()} | hash: ${block.hash} | new txs: ${ - transactions - transactionCacheMisses - }/${block.transactions.length.toString()} (${transactionCacheMisses} cache misses) | nodes: ${nodeAcceptances + ).toISOString()} | hash: ${ + block.hash + } | new txs: ${savedTransactionCount}/${block.transactions.length.toString()} (${transactionCacheMisses} cache misses) | nodes: ${nodeAcceptances .map((acceptance) => acceptance.nodeName) .join(', ')}`; + const blockInsertLog = `Inserting block ${ + block.height + } with ${savedTransactionCount} new transaction${ + savedTransactionCount === 1 ? '' : 's' + } for ${nodeAcceptances + .map((acceptance) => acceptance.nodeName) + .join(', ')} took ${formatDurationSeconds( + durationMs + )} (${formatTransactionDuration( + savedTransactionCount, + durationMs + )}, ${formatTransactionRate(savedTransactionCount, durationMs)})`; if (isHistoricalSync) { this.logger.debug(blockSyncLog); + this.logger.trace(blockInsertLog); } else { this.logger.info(blockSyncLog); + this.logger.debug(blockInsertLog); } nodeAcceptances.forEach((acceptance) => { @@ -1417,6 +2036,11 @@ export class Agent { blockTimestampToDate(block.timestamp) ); }); + const currentRepair = this.currentIncompleteBlockRepair; + if (currentRepair?.hash === block.hash) { + currentRepair.resolve(); + this.currentIncompleteBlockRepair = undefined; + } this.blockBuffer.removeBlock(block); } @@ -1430,12 +2054,16 @@ export class Agent { firstHeight: number, nodeName: string ) { - removeStaleBlocksForNode(this.nodes[nodeName]!.internalId!, staleChain) + const node = this.nodes[nodeName]!; + node.syncState?.blockReorganizationAtHeight(firstHeight); + removeStaleBlocksForNode(node.internalId!, staleChain) .then(() => { this.logger.info( staleChain, `${nodeName}: re-organization detected beginning at height: ${firstHeight}. The following stale blocks were removed:` ); + this.mempoolResync.handleReorganization(nodeName); + this.scheduleBlockBufferFill(); }) .catch((err) => { this.logger.error(err); @@ -1500,7 +2128,7 @@ export class Agent { txCacheItem.db = true; this.transactionCache.set(transactionHash, txCacheItem); this.logger.trace( - `Marked transaction saved to DB - hash: ${transactionHash}` + `Marked transaction as saved to DB - hash: ${transactionHash}` ); } @@ -1542,7 +2170,14 @@ export class Agent { .join(', ')} - hash: ${tx.hash}` ); // TODO: collect statistics on save speed + const startTime = Date.now(); await saveTransactionForNodes(tx, validations); + const durationMs = Date.now() - startTime; + this.logger.debug( + `Inserting mempool tx ${ + tx.hash + } for node ${nodeName} took ${formatDurationSeconds(durationMs)}` + ); this.markTransactionSavedToDb(tx.hash); } } @@ -1752,8 +2387,19 @@ export class Agent { return this.shutdownPromise; } this.willShutdown = true; + this.mempoolResync.stop(); clearInterval(eventLoopDurationInterval); clearInterval(this.heartbeatInterval); + if (this.incompleteBlockRepairTimeout !== undefined) { + clearTimeout(this.incompleteBlockRepairTimeout); + } + if (this.mempoolTransactionExpirationScanTimeout !== undefined) { + clearTimeout(this.mempoolTransactionExpirationScanTimeout); + } + this.mempoolTransactionExpirationTimers.forEach((timeout) => { + clearTimeout(timeout); + }); + this.mempoolTransactionExpirationTimers.clear(); Object.values(this.nodes).forEach((connection) => { connection.disconnect(); }); diff --git a/src/bitcore.spec.ts b/src/bitcore.spec.ts new file mode 100644 index 00000000..96bafd8c --- /dev/null +++ b/src/bitcore.spec.ts @@ -0,0 +1,197 @@ +import type { + BitcoreBlock, + BitcoreTransaction, +} from '@chaingraph/bitcore-p2p-cash'; +import test from 'ava'; + +import { + bitcoreBlockToChaingraphBlock, + bitcoreTransactionToChaingraphTransaction, + decodeBlockRetainingWireBytes, + messages, +} from './bitcore.js'; +import { + Block, + chipnetCashTokensTx, + chipnetCashTokensTxHash, + generateMockchain, + genesisBlockRaw, + halTxHash, + halTxRaw, + halTxSpent, + halTxSpentRaw, + testnetGenesisBlockRaw, + Transaction, +} from './e2e/e2e.spec.mockchain.helper.js'; +import type { + ChaingraphBlock, + ChaingraphTransaction, +} from './types/chaingraph.js'; + +/** + * The implementation prior to retaining wire bytes (re-serializes every + * transaction and the full block), kept as a reference for parity tests. + */ +const referenceTransactionConversion = ( + bitcoreTransaction: BitcoreTransaction +): ChaingraphTransaction => { + const txObject = bitcoreTransaction.toObject(); + const isCoinbase = bitcoreTransaction.isCoinbase(); + return { + hash: txObject.hash, + inputs: txObject.inputs.map((input) => ({ + outpointIndex: input.outputIndex, + outpointTransactionHash: input.prevTxId, + sequenceNumber: input.sequenceNumber, + unlockingBytecode: input.script, + })), + isCoinbase, + locktime: txObject.nLockTime, + outputs: txObject.outputs.map((output) => ({ + fungibleTokenAmount: + output.tokenData === undefined + ? undefined + : BigInt(output.tokenData.amount), + lockingBytecode: output.script, + nonfungibleTokenCapability: output.tokenData?.nft?.capability, + nonfungibleTokenCommitment: output.tokenData?.nft?.commitment, + tokenCategory: output.tokenData?.category, + valueSatoshis: BigInt(output.satoshis), + })), + sizeBytes: bitcoreTransaction.toBuffer().length, + version: txObject.version, + }; +}; + +const referenceBlockConversion = ( + bitcoreBlock: BitcoreBlock, + height: number +): ChaingraphBlock => { + const bitcoreBlockHeader = bitcoreBlock.header.toObject(); + return { + bits: bitcoreBlockHeader.bits, + hash: bitcoreBlockHeader.hash, + height, + merkleRoot: bitcoreBlockHeader.merkleRoot.toString('hex'), + nonce: bitcoreBlockHeader.nonce, + previousBlockHash: bitcoreBlockHeader.prevHash.toString('hex'), + sizeBytes: bitcoreBlock.toBuffer().length, + timestamp: bitcoreBlockHeader.time, + transactions: bitcoreBlock.transactions.map(referenceTransactionConversion), + version: bitcoreBlockHeader.version, + }; +}; + +const realTransactions = [halTxRaw, halTxSpentRaw, chipnetCashTokensTx]; + +/** + * A block containing real mainnet and chipnet (CashTokens) transactions. + */ +const realTransactionsBlock = () => { + const genesis = Block.fromString(genesisBlockRaw); + const coinbase = genesis.transactions[0]!.toObject(); + return Block.fromObject({ + header: { + bits: 0x1d00ffff, + merkleRoot: + '0000000000000000000000000000000000000000000000000000000000000000', + nonce: 1, + prevHash: genesis.header.hash, + time: 1231006506, + version: 1, + }, + transactions: [ + coinbase, + ...realTransactions.map((raw) => new Transaction(raw).toObject()), + ], + }); +}; + +const encodedTestBlocks = () => [ + Buffer.from(genesisBlockRaw, 'hex'), + Buffer.from(testnetGenesisBlockRaw, 'hex'), + Buffer.from(realTransactionsBlock().toBuffer()), + ...generateMockchain({ + length: 50, + previousBlockHash: Block.fromString(genesisBlockRaw).header.hash, + }).map((block) => Buffer.from(block.toBuffer())), +]; + +test('decodeBlockRetainingWireBytes: decodes like Block.fromBuffer', (t) => { + encodedTestBlocks().forEach((encoded) => { + const original = Buffer.from(encoded); + const block = decodeBlockRetainingWireBytes(encoded); + t.deepEqual(encoded, original, 'does not modify the encoded block'); + t.deepEqual(block.toObject(), Block.fromBuffer(original).toObject()); + t.deepEqual( + decodeBlockRetainingWireBytes(encoded).toObject(), + Block.fromBuffer(encoded).toObject() + ); + }); +}); + +test('bitcoreBlockToChaingraphBlock: matches the re-serializing implementation', (t) => { + /** + * Note: bitcore's `Block.fromBuffer` modifies the provided buffer (token + * categories are reversed in place), so each decoding gets a fresh copy. + */ + encodedTestBlocks().forEach((encoded, height) => { + const expected = referenceBlockConversion( + Block.fromBuffer(Buffer.from(encoded)), + height + ); + t.is(expected.sizeBytes, encoded.length); + t.deepEqual( + bitcoreBlockToChaingraphBlock( + decodeBlockRetainingWireBytes(Buffer.from(encoded)), + height + ), + expected, + 'with wire bytes' + ); + t.deepEqual( + bitcoreBlockToChaingraphBlock( + messages.Block.fromBuffer(Buffer.from(encoded)).block, + height + ), + expected, + 'decoded by Messages' + ); + t.deepEqual( + bitcoreBlockToChaingraphBlock( + Block.fromBuffer(Buffer.from(encoded)), + height + ), + expected, + 'without wire bytes' + ); + }); +}); + +test('bitcoreBlockToChaingraphBlock: real transaction hashes', (t) => { + const block = bitcoreBlockToChaingraphBlock( + decodeBlockRetainingWireBytes( + Buffer.from(realTransactionsBlock().toBuffer()) + ), + 1 + ); + t.deepEqual( + block.transactions.slice(1).map((tx) => tx.hash), + [halTxHash, halTxSpent, chipnetCashTokensTxHash] + ); + t.true(block.transactions[0]!.isCoinbase); + t.true( + block.transactions[3]!.outputs.some( + (output) => output.tokenCategory !== undefined + ) + ); +}); + +test('bitcoreTransactionToChaingraphTransaction: matches the re-serializing implementation', (t) => { + realTransactions.forEach((raw) => { + t.deepEqual( + bitcoreTransactionToChaingraphTransaction(new Transaction(raw)), + referenceTransactionConversion(new Transaction(raw)) + ); + }); +}); diff --git a/src/bitcore.ts b/src/bitcore.ts index ee0a00ff..d523e0c7 100644 --- a/src/bitcore.ts +++ b/src/bitcore.ts @@ -1,3 +1,5 @@ +import crypto, { createHash } from 'crypto'; + import type { BitcoreBlock, BitcoreTransaction, @@ -9,34 +11,157 @@ import type { ChaingraphTransaction, } from './types/chaingraph.js'; +const { internalBitcore } = bitcoreP2pCash; + +/** + * The serialized block as received from the P2P network, with the boundaries + * of each transaction within it. Used to take block/transaction sizes and + * transaction hashes directly from the wire bytes, rather than re-serializing + * each transaction (and the full block) with bitcore. + */ +interface BlockWireBytes { + /** + * The serialized block. + */ + encoded: Buffer; + /** + * The offset of each transaction within `encoded`, followed by the offset + * at which the final transaction ends (`transactions.length + 1` items). + */ + transactionBoundaries: number[]; +} + +const blockWireBytes = new WeakMap(); + +/** + * Decode a serialized block, retaining the serialized bytes and the boundaries + * of each transaction for use by {@link bitcoreBlockToChaingraphBlock}. The + * returned block is identical to the result of bitcore's `Block.fromBuffer`. + */ +export const decodeBlockRetainingWireBytes = (encoded: Buffer) => { + /** + * Bitcore decodes from a copy: its token prefix decoding reverses each token + * category in place within the buffer being read, which would otherwise + * corrupt the retained wire bytes. + */ + const reader = new internalBitcore.encoding.BufferReader( + Buffer.from(encoded) + ); + const header = internalBitcore.BlockHeader.fromBufferReader(reader); + const transactionCount = reader.readVarintNum(); + const transactions: BitcoreTransaction[] = []; + const transactionBoundaries: number[] = []; + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let index = 0; index < transactionCount; index += 1) { + transactionBoundaries.push(reader.pos); + transactions.push( + new internalBitcore.Transaction().fromBufferReader(reader) + ); + } + transactionBoundaries.push(reader.pos); + const block = new internalBitcore.Block({ header, transactions }); + blockWireBytes.set(block, { encoded, transactionBoundaries }); + return block; +}; + +/** + * A bitcore `Block` constructor which decodes blocks using + * {@link decodeBlockRetainingWireBytes}. Provided to bitcore-p2p-cash's + * `Messages` so blocks received from peers retain their wire bytes. + */ +export class BlockRetainingWireBytes extends internalBitcore.Block { + static override fromBuffer = decodeBlockRetainingWireBytes; +} + +/** + * Create the bitcore-p2p-cash `Messages` used to communicate with a peer on + * the provided network. Received blocks retain their wire bytes. + */ +export const createPeerMessages = (networkName: string) => + new bitcoreP2pCash.Messages({ + // eslint-disable-next-line @typescript-eslint/naming-convention + Block: BlockRetainingWireBytes, + network: internalBitcore.Networks.get(networkName), + }); + +/** + * Node.js' one-shot `crypto.hash` (Node.js v20.12+, v21.7+) avoids creating a + * `Hash` object per call. (Not yet included in this project's `@types/node`.) + */ +const oneShotHash = ( + crypto as unknown as { + hash?: (algorithm: string, data: Uint8Array, encoding: 'buffer') => Buffer; + } +).hash; + +const sha256 = + oneShotHash === undefined + ? (data: Uint8Array) => createHash('sha256').update(data).digest() + : (data: Uint8Array) => oneShotHash('sha256', data, 'buffer'); + +/** + * Compute the transaction hash (A.K.A. "transaction ID") of an encoded transaction: + * the double-SHA256 hash, in user-interface (reversed) byte order, hex-encoded. + */ +export const encodedTransactionToHash = (encodedTransaction: Uint8Array) => + sha256(sha256(encodedTransaction)).reverse().toString('hex'); + +const coinbaseOutpointIndex = 0xffffffff; +const hashLength = 32; +const coinbaseOutpointTransactionHash = '00'.repeat(hashLength); + +/** + * Convert a bitcore transaction to the Chaingraph format. + * @param bitcoreTransaction - the bitcore transaction + * @param encodedTransaction - the serialized transaction, if available (e.g. + * as received from the network); if not provided, the transaction is + * re-serialized + */ export const bitcoreTransactionToChaingraphTransaction = ( - bitcoreTransaction: BitcoreTransaction + bitcoreTransaction: BitcoreTransaction, + encodedTransaction: Uint8Array = bitcoreTransaction.toBuffer() ): ChaingraphTransaction => { - const txObject = bitcoreTransaction.toObject(); - const isCoinbase = bitcoreTransaction.isCoinbase(); + /** + * Fields are read directly rather than via bitcore's `Input.toObject`, which + * also decodes and stringifies each unlocking bytecode (`scriptString`). + */ + const inputs = bitcoreTransaction.inputs.map((input) => ({ + outpointIndex: input.outputIndex, + outpointTransactionHash: input.prevTxId.toString('hex'), + sequenceNumber: input.sequenceNumber, + // eslint-disable-next-line no-underscore-dangle + unlockingBytecode: input._scriptBuffer.toString('hex'), + })); + /** + * Equivalent to bitcore's `Transaction.isCoinbase`, reusing the hex-encoded + * outpoint transaction hash. + */ + const isCoinbase = + inputs.length === 1 && + inputs[0]!.outpointIndex === coinbaseOutpointIndex && + inputs[0]!.outpointTransactionHash === coinbaseOutpointTransactionHash; return { - hash: txObject.hash, - inputs: txObject.inputs.map((input) => ({ - outpointIndex: input.outputIndex, - outpointTransactionHash: input.prevTxId, - sequenceNumber: input.sequenceNumber, - unlockingBytecode: input.script, - })), + hash: encodedTransactionToHash(encodedTransaction), + inputs, isCoinbase, - locktime: txObject.nLockTime, - outputs: txObject.outputs.map((output) => ({ - fungibleTokenAmount: - output.tokenData === undefined - ? undefined - : BigInt(output.tokenData.amount), - lockingBytecode: output.script, - nonfungibleTokenCapability: output.tokenData?.nft?.capability, - nonfungibleTokenCommitment: output.tokenData?.nft?.commitment, - tokenCategory: output.tokenData?.category, - valueSatoshis: BigInt(output.satoshis), - })), - sizeBytes: bitcoreTransaction.toBuffer().length, - version: txObject.version, + locktime: bitcoreTransaction.nLockTime, + outputs: bitcoreTransaction.outputs.map((output) => { + const { tokenData } = output; + return { + fungibleTokenAmount: + tokenData === undefined + ? undefined + : BigInt(tokenData.amount.toString()), + // eslint-disable-next-line no-underscore-dangle + lockingBytecode: output._scriptBuffer.toString('hex'), + nonfungibleTokenCapability: tokenData?.nft?.capability, + nonfungibleTokenCommitment: tokenData?.nft?.commitment, + tokenCategory: tokenData?.category, + valueSatoshis: BigInt(output.satoshis), + }; + }), + sizeBytes: encodedTransaction.length, + version: bitcoreTransaction.version, }; }; @@ -45,6 +170,21 @@ export const bitcoreBlockToChaingraphBlock = ( height: number ): ChaingraphBlock => { const bitcoreBlockHeader = bitcoreBlock.header.toObject(); + const wireBytes = blockWireBytes.get(bitcoreBlock); + const transactions = + wireBytes === undefined + ? bitcoreBlock.transactions.map((transaction) => + bitcoreTransactionToChaingraphTransaction(transaction) + ) + : bitcoreBlock.transactions.map((transaction, index) => + bitcoreTransactionToChaingraphTransaction( + transaction, + wireBytes.encoded.subarray( + wireBytes.transactionBoundaries[index], + wireBytes.transactionBoundaries[index + 1] + ) + ) + ); return { bits: bitcoreBlockHeader.bits, hash: bitcoreBlockHeader.hash, @@ -52,13 +192,19 @@ export const bitcoreBlockToChaingraphBlock = ( merkleRoot: bitcoreBlockHeader.merkleRoot.toString('hex'), nonce: bitcoreBlockHeader.nonce, previousBlockHash: bitcoreBlockHeader.prevHash.toString('hex'), - sizeBytes: bitcoreBlock.toBuffer().length, + sizeBytes: + wireBytes === undefined + ? bitcoreBlock.toBuffer().length + : wireBytes.transactionBoundaries[ + wireBytes.transactionBoundaries.length - 1 + ]!, timestamp: bitcoreBlockHeader.time, - transactions: bitcoreBlock.transactions.map( - bitcoreTransactionToChaingraphTransaction - ), + transactions, version: bitcoreBlockHeader.version, }; }; -export const messages = new bitcoreP2pCash.Messages(); +export const messages = new bitcoreP2pCash.Messages({ + // eslint-disable-next-line @typescript-eslint/naming-convention + Block: BlockRetainingWireBytes, +}); diff --git a/src/components/block-copy-rows.spec.ts b/src/components/block-copy-rows.spec.ts new file mode 100644 index 00000000..2ce5a898 --- /dev/null +++ b/src/components/block-copy-rows.spec.ts @@ -0,0 +1,192 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import test from 'ava'; + +import type { + ChaingraphBlock, + ChaingraphTransaction, +} from '../types/chaingraph.js'; + +import { + copyStageTableSql, + createStageTablesSql, + encodeStageBlockTransactions, + encodeStageInputs, + encodeStageOutputs, + encodeStageTransactions, + stageTableColumns, +} from './block-copy-rows.js'; +import { decodeBinaryCopy } from './pg-binary-copy.js'; + +const hash = (byte: string) => byte.repeat(32); +const int8 = (field: Buffer | null | undefined) => field!.readBigInt64BE(0); +const hex = (field: Buffer | null | undefined) => field!.toString('hex'); + +const coinbase: ChaingraphTransaction = { + hash: hash('aa'), + inputs: [ + { + outpointIndex: 0xffffffff, + outpointTransactionHash: hash('00'), + sequenceNumber: 0xffffffff, + unlockingBytecode: '03a0bb0d', + }, + ], + isCoinbase: true, + locktime: 0, + outputs: [{ lockingBytecode: '', valueSatoshis: 625000000n }], + sizeBytes: 100, + version: 1, +}; + +const tokenTransaction: ChaingraphTransaction = { + hash: hash('bb'), + inputs: [ + { + outpointIndex: 1, + outpointTransactionHash: hash('cc'), + sequenceNumber: 0, + unlockingBytecode: '', + }, + { + outpointIndex: 0, + outpointTransactionHash: hash('dd'), + sequenceNumber: 0xfffffffe, + unlockingBytecode: '0051', + }, + ], + isCoinbase: false, + locktime: 500000000, + outputs: [ + { + fungibleTokenAmount: 9223372036854775807n, + lockingBytecode: '76a914', + nonfungibleTokenCapability: 'minting', + nonfungibleTokenCommitment: '', + tokenCategory: hash('ee'), + valueSatoshis: 1000n, + }, + { + lockingBytecode: '6a', + nonfungibleTokenCapability: 'none', + nonfungibleTokenCommitment: '01020304', + tokenCategory: hash('ff'), + valueSatoshis: 0n, + }, + { + fungibleTokenAmount: 1n, + lockingBytecode: 'aa20', + tokenCategory: hash('11'), + valueSatoshis: 800n, + }, + ], + sizeBytes: 300, + version: 2, +}; + +test('encodeStageTransactions: one row per transaction in column order', (t) => { + const rows = decodeBinaryCopy( + encodeStageTransactions([coinbase, tokenTransaction]) + ); + t.is(rows.length, 2); + t.is(rows[0]!.length, stageTableColumns.chaingraph_stage_transaction.length); + t.deepEqual( + rows.map((row) => [ + hex(row[0]), + int8(row[1]), + int8(row[2]), + int8(row[3]), + row[4]![0], + ]), + [ + [hash('aa'), 1n, 0n, 100n, 1], + [hash('bb'), 2n, 500000000n, 300n, 0], + ] + ); +}); + +test('encodeStageInputs: indexes, uint32 sequence numbers and empty bytecode', (t) => { + const rows = decodeBinaryCopy( + encodeStageInputs([coinbase, tokenTransaction]) + ); + t.deepEqual( + rows.map((row) => [ + hex(row[0]), + int8(row[1]), + int8(row[2]), + int8(row[3]), + hex(row[4]), + hex(row[5]), + ]), + [ + [hash('aa'), 0n, 4294967295n, 4294967295n, hash('00'), '03a0bb0d'], + [hash('bb'), 0n, 1n, 0n, hash('cc'), ''], + [hash('bb'), 1n, 0n, 4294967294n, hash('dd'), '0051'], + ] + ); +}); + +test('encodeStageOutputs: token fields, enum labels and NULLs', (t) => { + const rows = decodeBinaryCopy( + encodeStageOutputs([coinbase, tokenTransaction]) + ); + t.is(rows[0]!.length, stageTableColumns.chaingraph_stage_output.length); + t.deepEqual( + rows.map((row) => [ + hex(row[0]), + int8(row[1]), + int8(row[2]), + hex(row[3]), + row[4] === null ? null : hex(row[4]), + row[5] === null ? null : int8(row[5]), + row[6] === null ? null : row[6]!.toString('utf8'), + row[7] === null ? null : hex(row[7]), + ]), + [ + [hash('aa'), 0n, 625000000n, '', null, null, null, null], + [ + hash('bb'), + 0n, + 1000n, + '76a914', + hash('ee'), + 9223372036854775807n, + 'minting', + '', + ], + [hash('bb'), 1n, 0n, '6a', hash('ff'), null, 'none', '01020304'], + [hash('bb'), 2n, 800n, 'aa20', hash('11'), 1n, null, null], + ] + ); +}); + +test('encodeStage*: empty transaction lists encode zero rows', (t) => { + t.deepEqual(decodeBinaryCopy(encodeStageTransactions([])), []); + t.deepEqual(decodeBinaryCopy(encodeStageInputs([])), []); + t.deepEqual(decodeBinaryCopy(encodeStageOutputs([])), []); +}); + +test('encodeStageBlockTransactions: every transaction with its block index', (t) => { + const block = { + transactions: [coinbase, tokenTransaction], + } as unknown as ChaingraphBlock; + const rows = decodeBinaryCopy(encodeStageBlockTransactions(block)); + t.deepEqual( + rows.map((row) => [hex(row[0]), int8(row[1])]), + [ + [hash('aa'), 0n], + [hash('bb'), 1n], + ] + ); +}); + +test('stage table SQL', (t) => { + t.is( + copyStageTableSql('chaingraph_stage_block_transaction'), + 'COPY pg_temp.chaingraph_stage_block_transaction (hash, transaction_index) FROM STDIN (FORMAT binary)' + ); + t.regex( + createStageTablesSql, + /CREATE TEMP TABLE IF NOT EXISTS chaingraph_stage_output \(transaction_hash bytea, .*nonfungible_token_capability enum_nonfungible_token_capability, nonfungible_token_commitment bytea\) ON COMMIT DELETE ROWS;/u + ); + t.is(createStageTablesSql.split('\n').length, 4); +}); diff --git a/src/components/block-copy-rows.ts b/src/components/block-copy-rows.ts new file mode 100644 index 00000000..02897df8 --- /dev/null +++ b/src/components/block-copy-rows.ts @@ -0,0 +1,200 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +/** + * Binary COPY encoders for the per-connection staging tables used by the + * `copy` write path (`CHAINGRAPH_WRITE_PATH=copy`, see `saveBlock` in + * `db.ts`). Column order here must match `stageTableColumns`. + */ +import type { + ChaingraphBlock, + ChaingraphTransaction, +} from '../types/chaingraph.js'; + +import { BinaryCopyWriter } from './pg-binary-copy.js'; + +/** + * Temporary staging tables, created once per pooled connection. Rows are + * removed at every COMMIT (and discarded by ROLLBACK), so each block's DB + * transaction starts with empty tables. + * + * Names are prefixed to avoid shadowing permanent tables (`pg_temp` is + * searched first) and always referenced as `pg_temp.`. + */ +export const stageTableColumns = { + /* eslint-disable camelcase, @typescript-eslint/naming-convention */ + chaingraph_stage_block_transaction: [ + ['hash', 'bytea'], + ['transaction_index', 'bigint'], + ], + chaingraph_stage_input: [ + ['transaction_hash', 'bytea'], + ['input_index', 'bigint'], + ['outpoint_index', 'bigint'], + ['sequence_number', 'bigint'], + ['outpoint_transaction_hash', 'bytea'], + ['unlocking_bytecode', 'bytea'], + ], + chaingraph_stage_output: [ + ['transaction_hash', 'bytea'], + ['output_index', 'bigint'], + ['value_satoshis', 'bigint'], + ['locking_bytecode', 'bytea'], + ['token_category', 'bytea'], + ['fungible_token_amount', 'bigint'], + ['nonfungible_token_capability', 'enum_nonfungible_token_capability'], + ['nonfungible_token_commitment', 'bytea'], + ], + chaingraph_stage_transaction: [ + ['hash', 'bytea'], + ['version', 'bigint'], + ['locktime', 'bigint'], + ['size_bytes', 'bigint'], + ['is_coinbase', 'boolean'], + ], + /* eslint-enable camelcase, @typescript-eslint/naming-convention */ +} as const; + +export type StageTable = keyof typeof stageTableColumns; + +export const createStageTablesSql = Object.entries(stageTableColumns) + .map( + ([table, columns]) => + `CREATE TEMP TABLE IF NOT EXISTS ${table} (${columns + .map(([name, type]) => `${name} ${type}`) + .join(', ')}) ON COMMIT DELETE ROWS;` + ) + .join('\n'); + +const buildCopyStageTableSql = (table: StageTable) => + `COPY pg_temp.${table} (${stageTableColumns[table] + .map(([name]) => name) + .join(', ')}) FROM STDIN (FORMAT binary)`; + +const copyStageTableStatements = Object.fromEntries( + Object.keys(stageTableColumns).map((table) => [ + table, + buildCopyStageTableSql(table as StageTable), + ]) +) as { [table in StageTable]: string }; + +export const copyStageTableSql = (table: StageTable) => + copyStageTableStatements[table]; + +const hashBytes = 32; +const copyHeaderAndTrailerBytes = 64; +const transactionRowBytes = 2 + 5 * 4 + hashBytes + 3 * 8 + 1; +const blockTransactionRowBytes = 2 + 2 * 4 + hashBytes + 8; +const hexCharsPerByte = 2; +const inputRowFixedBytes = 2 + 6 * 4 + 2 * hashBytes + 3 * 8; +const outputRowFixedBytes = 2 + 8 * 4 + hashBytes + 2 * 8; + +export const encodeStageTransactions = ( + transactions: ChaingraphTransaction[] +) => { + const writer = new BinaryCopyWriter( + transactions.length * transactionRowBytes + copyHeaderAndTrailerBytes + ); + transactions.forEach((transaction) => { + writer + .startRow(stageTableColumns.chaingraph_stage_transaction.length) + .hexBytea(transaction.hash) + .int8(transaction.version) + .int8(transaction.locktime) + .int8(transaction.sizeBytes) + .boolean(transaction.isCoinbase); + }); + return writer.finish(); +}; + +export const encodeStageInputs = (transactions: ChaingraphTransaction[]) => { + const expectedBytes = transactions.reduce( + (total, transaction) => + total + + transaction.inputs.reduce( + (sum, input) => + sum + + inputRowFixedBytes + + input.unlockingBytecode.length / hexCharsPerByte, + 0 + ), + copyHeaderAndTrailerBytes + ); + const writer = new BinaryCopyWriter(expectedBytes); + transactions.forEach((transaction) => { + transaction.inputs.forEach((input, inputIndex) => { + writer + .startRow(stageTableColumns.chaingraph_stage_input.length) + .hexBytea(transaction.hash) + .int8(inputIndex) + .int8(input.outpointIndex) + .int8(input.sequenceNumber) + .hexBytea(input.outpointTransactionHash) + .hexBytea(input.unlockingBytecode); + }); + }); + return writer.finish(); +}; + +export const encodeStageOutputs = (transactions: ChaingraphTransaction[]) => { + const expectedBytes = transactions.reduce( + (total, transaction) => + total + + transaction.outputs.reduce( + (sum, output) => + sum + + outputRowFixedBytes + + output.lockingBytecode.length / hexCharsPerByte, + 0 + ), + copyHeaderAndTrailerBytes + ); + const writer = new BinaryCopyWriter(expectedBytes); + transactions.forEach((transaction) => { + transaction.outputs.forEach((output, outputIndex) => { + writer + .startRow(stageTableColumns.chaingraph_stage_output.length) + .hexBytea(transaction.hash) + .int8(outputIndex) + .int8(output.valueSatoshis) + .hexBytea(output.lockingBytecode); + if (output.tokenCategory === undefined) { + writer.null(); + } else { + writer.hexBytea(output.tokenCategory); + } + if (output.fungibleTokenAmount === undefined) { + writer.null(); + } else { + writer.int8(output.fungibleTokenAmount); + } + if (output.nonfungibleTokenCapability === undefined) { + writer.null(); + } else { + writer.text(output.nonfungibleTokenCapability); + } + if (output.nonfungibleTokenCommitment === undefined) { + writer.null(); + } else { + writer.hexBytea(output.nonfungibleTokenCommitment); + } + }); + }); + return writer.finish(); +}; + +/** + * Every transaction in the block (including those already saved), with its + * index in the block. + */ +export const encodeStageBlockTransactions = (block: ChaingraphBlock) => { + const writer = new BinaryCopyWriter( + block.transactions.length * blockTransactionRowBytes + + copyHeaderAndTrailerBytes + ); + block.transactions.forEach((transaction, transactionIndex) => { + writer + .startRow(stageTableColumns.chaingraph_stage_block_transaction.length) + .hexBytea(transaction.hash) + .int8(transactionIndex); + }); + return writer.finish(); +}; diff --git a/src/components/block-tree.spec.ts b/src/components/block-tree.spec.ts index 378a9dda..09ee6324 100644 --- a/src/components/block-tree.spec.ts +++ b/src/components/block-tree.spec.ts @@ -1,8 +1,10 @@ /* eslint-disable @typescript-eslint/no-magic-numbers */ -import { range } from '@bitauth/libauth'; +import { hexToBin, range } from '@bitauth/libauth'; +import type { BitcoreBlockHeader } from '@chaingraph/bitcore-p2p-cash'; import test from 'ava'; +import { pino } from 'pino'; -import { selectHeaderLocatorHashes } from './block-tree.js'; +import { BlockTree, selectHeaderLocatorHashes } from './block-tree.js'; test('selectHeaderLocatorHashes', async (t) => { const testLength = 654247; @@ -33,3 +35,84 @@ test('selectHeaderLocatorHashes: `100`', async (t) => { const expected = [99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 86, 82, 74, 58, 26, 0]; t.deepEqual(selectHeaderLocatorHashes(headerChain), expected); }); + +/** + * Build a fake header-hash chain: the hash of each block encodes its branch + * and height, e.g. `blockHash('a', 3)`. + */ +const blockHash = (branch: string, height: number) => + `${branch}${height.toString(16).padStart(63, '0')}`; + +const createHeaders = (hashes: string[], previousHash: string) => + hashes.map((hash, index) => ({ + hash, + prevHash: Buffer.from( + hexToBin(index === 0 ? previousHash : hashes[index - 1]!).reverse() + ), + })) as unknown as BitcoreBlockHeader[]; + +/** + * Create a `BlockTree` for node `n` restored to the provided chain, recording + * each `onStaleBlocks` call. + */ +const createBlockTree = (chain: string[]) => { + const staleCalls: { staleChain: string[]; firstHeight: number }[] = []; + const blockTree = new BlockTree({ + genesisBlockByNode: { n: [chain[0]!] }, + logger: pino({ level: 'silent' }), + onStaleBlocks: (staleChain, firstHeight) => { + staleCalls.push({ firstHeight, staleChain }); + }, + }); + blockTree.restoreChainForNode('n', chain); + return { blockTree, staleCalls }; +}; + +const readChain = (blockTree: BlockTree) => + range(blockTree.getBestHeights().n! + 1).map((height) => + blockTree.getBlockHeaderHash('n', height) + ); + +/** + * Genesis (height 0) followed by blocks `1` through `tip` of `branch`, with + * heights up to `forkHeight` shared with branch `a`. + */ +const branchChain = (branch: string, forkHeight: number, tip: number) => + range(tip + 1).map((height) => + blockHash(height <= forkHeight ? 'a' : branch, height) + ); + +test('BlockTree.updateHeaders: extends the chain without stale blocks', (t) => { + const { blockTree, staleCalls } = createBlockTree(branchChain('a', 10, 10)); + const newHashes = branchChain('a', 13, 13).slice(11); + blockTree.updateHeaders('n', createHeaders(newHashes, blockHash('a', 10))); + t.deepEqual(staleCalls, []); + t.deepEqual(readChain(blockTree), branchChain('a', 13, 13)); +}); + +test('BlockTree.updateHeaders: shallow re-organization starting after the fork point', (t) => { + const { blockTree, staleCalls } = createBlockTree(branchChain('a', 10, 10)); + const newHashes = branchChain('b', 8, 11).slice(9); + blockTree.updateHeaders('n', createHeaders(newHashes, blockHash('a', 8))); + t.deepEqual(staleCalls, [ + { firstHeight: 9, staleChain: [blockHash('a', 9), blockHash('a', 10)] }, + ]); + t.deepEqual(readChain(blockTree), branchChain('b', 8, 11)); +}); + +test('BlockTree.updateHeaders: deep re-organization with headers starting below the fork point only marks diverging blocks as stale', (t) => { + const { blockTree, staleCalls } = createBlockTree(branchChain('a', 105, 105)); + /** + * As when a node answers `getheaders` from the latest locator hash in its + * active chain (here genesis): heights 1-5 are shared with the known chain. + */ + const newHashes = branchChain('b', 5, 106).slice(1); + blockTree.updateHeaders('n', createHeaders(newHashes, blockHash('a', 0))); + t.deepEqual(staleCalls, [ + { + firstHeight: 6, + staleChain: range(100, 6).map((height) => blockHash('a', height)), + }, + ]); + t.deepEqual(readChain(blockTree), branchChain('b', 5, 106)); +}); diff --git a/src/components/block-tree.ts b/src/components/block-tree.ts index 602c5950..120ff6ac 100644 --- a/src/components/block-tree.ts +++ b/src/components/block-tree.ts @@ -206,16 +206,28 @@ export class BlockTree { return this.getLocatorForNode(nodeName); } const newHashes = headers.map((header) => header.hash); + /** + * A `headers` reply may begin below the fork point (e.g. when the fork + * block is not in our locator), so leading headers can match blocks we + * already have. Only headers from the first differing height replace + * (and make stale) existing blocks. + */ + const firstDivergingIndex = newHashes.findIndex( + (hash, index) => chain[firstHeight + index] !== hash + ); + const sharedCount = + firstDivergingIndex === -1 ? newHashes.length : firstDivergingIndex; + const divergeHeight = firstHeight + sharedCount; const staleHashes = chain.splice( - firstHeight, - newHashes.length, - ...newHashes + divergeHeight, + newHashes.length - sharedCount, + ...newHashes.slice(sharedCount) ); if (staleHashes.length !== 0) { this.logger.warn( `${nodeName}: removed ${staleHashes.length} stale block(s) from the block tree.` ); - this.onStaleBlocks(staleHashes, firstHeight, nodeName); + this.onStaleBlocks(staleHashes, divergeHeight, nodeName); } this.logger.debug( `${nodeName}: added ${ diff --git a/src/components/db-utils.ts b/src/components/db-utils.ts index dfdacd0e..e1f03e51 100644 --- a/src/components/db-utils.ts +++ b/src/components/db-utils.ts @@ -4,6 +4,7 @@ export const indexDefinitions = { block_inclusions_index: /* sql */ `CREATE INDEX block_inclusions_index ON block_transaction USING btree (transaction_internal_id);`, output_search_index: /* sql */ `CREATE INDEX output_search_index ON output USING btree (substring(locking_bytecode, 0, 26));`, spent_by_index: /* sql */ `CREATE INDEX spent_by_index ON input USING btree (outpoint_transaction_hash, outpoint_index);`, + token_category_index: /* sql */ `CREATE INDEX token_category_index ON output USING btree (token_category);`, }; /* eslint-enable camelcase, @typescript-eslint/naming-convention */ diff --git a/src/components/mempool-resync.spec.ts b/src/components/mempool-resync.spec.ts new file mode 100644 index 00000000..f5e51d01 --- /dev/null +++ b/src/components/mempool-resync.spec.ts @@ -0,0 +1,276 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import bitcoreP2pCash from '@chaingraph/bitcore-p2p-cash'; +import type { Message } from '@chaingraph/bitcore-p2p-cash'; +import test from 'ava'; +import LRU from 'lru-cache'; + +import type { + MempoolResyncCacheItem, + MempoolResyncOptions, + MempoolResyncTrigger, +} from './mempool-resync.js'; +import { + forgetSavedAcknowledgementsFromNode, + MempoolResync, +} from './mempool-resync.js'; + +interface MockNode { + name: string; + peer: InstanceType; + ready: boolean; + sentMessages: Message[]; +} + +const createMockNode = (name: string): MockNode => { + const sentMessages: Message[] = []; + const peer = new bitcoreP2pCash.Peer({}); + peer.sendMessage = (message) => { + sentMessages.push(message); + }; + return { name, peer, ready: true, sentMessages }; +}; + +const commandStart = 4; +const commandEnd = 16; +const commandsSent = (node: MockNode) => + node.sentMessages.map((message) => + message + .toBuffer() + .subarray(commandStart, commandEnd) + .toString('latin1') + .replace(/\0+$/u, '') + ); + +const setup = (options: Partial> = {}) => { + const nodes = { + node1: createMockNode('node1'), + node2: createMockNode('node2'), + }; + const transactionCache = new LRU>({ + max: 10, + }); + const clock = { now: 0 }; + const timers: { callback: () => void; dueAt: number; cleared: boolean }[] = + []; + const advance = (ms: number) => { + clock.now += ms; + timers + .filter((timer) => !timer.cleared && timer.dueAt <= clock.now) + .forEach((timer) => { + timer.cleared = true; + timer.callback(); + }); + }; + const requestsSent: [string, MempoolResyncTrigger[]][] = []; + const mempoolResync = new MempoolResync({ + clearTimer: (timer) => { + (timer as (typeof timers)[number]).cleared = true; + }, + enabled: true, + getReadyNode: (nodeName) => { + const node = nodes[nodeName as keyof typeof nodes]; + return node.ready ? node : undefined; + }, + minimumIntervalMs: 1000, + now: () => clock.now, + onRequestSent: (nodeName, triggers) => { + requestsSent.push([nodeName, triggers]); + }, + setTimer: (callback, ms) => { + const timer = { callback, cleared: false, dueAt: clock.now + ms }; + timers.push(timer); + return timer; + }, + transactionCache, + ...options, + }); + return { + advance, + mempoolResync, + nodes, + requestsSent, + timers, + transactionCache, + }; +}; + +test('sends the BIP35 mempool wire message', (t) => { + const { mempoolResync, nodes } = setup(); + mempoolResync.enableMempoolTracking(['node1']); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + const wire = nodes.node1.sentMessages[0]!.toBuffer(); + t.is(wire.subarray(4, 16).toString('hex'), '6d656d706f6f6c0000000000'); + t.is(wire.readUInt32LE(16), 0); + t.is(wire.length, 24); +}); + +test('requests every node once mempool tracking is enabled', (t) => { + const { mempoolResync, nodes, requestsSent } = setup(); + mempoolResync.enableMempoolTracking(['node1', 'node2']); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + t.deepEqual(commandsSent(nodes.node2), ['mempool']); + t.deepEqual(requestsSent, [ + ['node1', ['mempool-tracking-enabled']], + ['node2', ['mempool-tracking-enabled']], + ]); +}); + +test('requests a node on (re)connect, but only after mempool tracking is enabled', (t) => { + const { advance, mempoolResync, nodes, requestsSent } = setup(); + mempoolResync.handleConnect('node1'); + t.deepEqual(commandsSent(nodes.node1), []); + mempoolResync.enableMempoolTracking([]); + mempoolResync.handleConnect('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + advance(1000); + mempoolResync.handleConnect('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool']); + t.deepEqual(requestsSent, [ + ['node1', ['connect']], + ['node1', ['connect']], + ]); +}); + +test('requests a node after a re-organization, but only after mempool tracking is enabled', (t) => { + const { mempoolResync, nodes, requestsSent } = setup(); + mempoolResync.handleReorganization('node2'); + t.deepEqual(commandsSent(nodes.node2), []); + mempoolResync.enableMempoolTracking([]); + mempoolResync.handleReorganization('node2'); + t.deepEqual(commandsSent(nodes.node1), []); + t.deepEqual(commandsSent(nodes.node2), ['mempool']); + t.deepEqual(requestsSent, [['node2', ['reorganization']]]); +}); + +test('sends nothing when disabled', (t) => { + const { mempoolResync, nodes, timers } = setup({ enabled: false }); + mempoolResync.enableMempoolTracking(['node1', 'node2']); + mempoolResync.handleConnect('node1'); + mempoolResync.handleReorganization('node2'); + t.deepEqual(commandsSent(nodes.node1), []); + t.deepEqual(commandsSent(nodes.node2), []); + t.is(timers.length, 0); +}); + +test('forgets saved acknowledgements from the node before each request', (t) => { + const { mempoolResync, nodes, transactionCache } = setup(); + const { node1, node2 } = nodes; + transactionCache.set('saved-both', { db: true, nodes: [node1, node2] }); + transactionCache.set('saved-node1', { db: true, nodes: [node1] }); + transactionCache.set('saved-node2', { db: true, nodes: [node2] }); + transactionCache.set('unsaved', { db: false, nodes: [node1, node2] }); + node1.peer.sendMessage = () => { + t.deepEqual(transactionCache.get('saved-node1')!.nodes, []); + }; + mempoolResync.handleConnect('node1'); + mempoolResync.enableMempoolTracking(['node1']); + t.deepEqual(transactionCache.get('saved-both'), { + db: true, + nodes: [node2], + }); + t.deepEqual(transactionCache.get('saved-node1'), { db: true, nodes: [] }); + t.deepEqual(transactionCache.get('saved-node2'), { + db: true, + nodes: [node2], + }); + t.deepEqual(transactionCache.get('unsaved'), { + db: false, + nodes: [node1, node2], + }); + t.is(transactionCache.size, 4); + t.plan(6); +}); + +test('forgetSavedAcknowledgementsFromNode never removes cache entries', (t) => { + const { nodes, transactionCache } = setup(); + transactionCache.set('a', { db: true, nodes: [nodes.node1] }); + forgetSavedAcknowledgementsFromNode(nodes.node1, transactionCache); + forgetSavedAcknowledgementsFromNode(nodes.node1, transactionCache); + t.deepEqual(transactionCache.get('a'), { db: true, nodes: [] }); + t.is(transactionCache.size, 1); +}); + +test('rate-limits each node, coalescing triggers into one deferred request', (t) => { + const { advance, mempoolResync, nodes, requestsSent, timers } = setup(); + mempoolResync.enableMempoolTracking(['node1', 'node2']); + advance(100); + mempoolResync.handleReorganization('node1'); + mempoolResync.handleConnect('node1'); + mempoolResync.handleReorganization('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + t.is(timers.length, 1); + advance(899); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + advance(1); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool']); + t.deepEqual(commandsSent(nodes.node2), ['mempool']); + t.deepEqual(requestsSent.slice(2), [ + ['node1', ['reorganization', 'connect']], + ]); + advance(500); + mempoolResync.handleReorganization('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool']); + advance(500); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool', 'mempool']); +}); + +test('a flapping node is requested at most once per interval', (t) => { + const { advance, mempoolResync, nodes } = setup(); + mempoolResync.enableMempoolTracking([]); + Array.from({ length: 100 }).forEach(() => { + mempoolResync.handleConnect('node1'); + advance(50); + }); + /** + * 100 reconnects over 5 seconds: sent at 0, 1000, 2000, 3000, 4000, and + * (coalescing the reconnects after 4000) 5000. + */ + t.is(commandsSent(nodes.node1).length, 6); +}); + +test('skips a deferred request if the node disconnected', (t) => { + const { advance, mempoolResync, nodes, requestsSent } = setup(); + mempoolResync.enableMempoolTracking(['node1']); + mempoolResync.handleReorganization('node1'); + nodes.node1.ready = false; + advance(1000); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + nodes.node1.ready = true; + mempoolResync.handleConnect('node1'); + t.deepEqual(commandsSent(nodes.node1), ['mempool', 'mempool']); + t.deepEqual(requestsSent, [ + ['node1', ['mempool-tracking-enabled']], + ['node1', ['connect']], + ]); +}); + +test('stop cancels deferred requests and ignores later triggers', (t) => { + const { advance, mempoolResync, nodes, timers } = setup(); + mempoolResync.enableMempoolTracking(['node1']); + mempoolResync.handleReorganization('node1'); + mempoolResync.stop(); + t.true(timers[0]!.cleared); + advance(5000); + mempoolResync.handleReorganization('node1'); + mempoolResync.handleConnect('node2'); + t.deepEqual(commandsSent(nodes.node1), ['mempool']); + t.deepEqual(commandsSent(nodes.node2), []); +}); + +test('uses real timers by default', async (t) => { + const node = createMockNode('node1'); + const mempoolResync = new MempoolResync({ + enabled: true, + getReadyNode: () => node, + minimumIntervalMs: 20, + transactionCache: [], + }); + mempoolResync.enableMempoolTracking(['node1']); + mempoolResync.handleReorganization('node1'); + t.deepEqual(commandsSent(node), ['mempool']); + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + t.deepEqual(commandsSent(node), ['mempool', 'mempool']); + mempoolResync.stop(); +}); diff --git a/src/components/mempool-resync.ts b/src/components/mempool-resync.ts new file mode 100644 index 00000000..6a3bbe46 --- /dev/null +++ b/src/components/mempool-resync.ts @@ -0,0 +1,206 @@ +import type { Peer } from '@chaingraph/bitcore-p2p-cash'; + +/** + * The events after which a node's mempool is requested. + */ +export type MempoolResyncTrigger = + | 'connect' + | 'mempool-tracking-enabled' + | 'reorganization'; + +export interface MempoolResyncNode { + peer: Pick; +} + +export interface MempoolResyncCacheItem { + db: boolean; + nodes: Node[]; +} + +export interface MempoolResyncOptions { + /** + * If `false`, no `mempool` requests are ever sent. + */ + enabled: boolean; + /** + * The minimum number of milliseconds between two `mempool` requests to the + * same node. Requests made during this window are coalesced into a single + * request sent when the window ends. + */ + minimumIntervalMs: number; + /** + * Return the node if it is connected and registered (ready to receive a + * `mempool` request and to have its announcements recorded), otherwise + * `undefined`. + */ + getReadyNode: (nodeName: string) => Node | undefined; + /** + * The agent's transaction cache. Before each request, the node is removed + * from the acknowledgements of already-saved transactions. + */ + transactionCache: Iterable<[string, MempoolResyncCacheItem]>; + /** + * Called after each `mempool` request is sent. + */ + onRequestSent?: (nodeName: string, triggers: MempoolResyncTrigger[]) => void; + now?: () => number; + setTimer?: (callback: () => void, ms: number) => unknown; + clearTimer?: (timer: unknown) => void; +} + +/** + * Remove `node` from the cached acknowledgements of every transaction which + * has already been saved to the database, so a later announcement of the + * transaction by this node is recorded again (rather than ignored as a + * duplicate). Acknowledgements of transactions not yet saved are kept: they + * are still needed to record validations when the transaction is saved. + */ +export const forgetSavedAcknowledgementsFromNode = ( + node: Node, + transactionCache: Iterable<[string, MempoolResyncCacheItem]> +) => { + // eslint-disable-next-line functional/no-loop-statement + for (const [, cacheItem] of transactionCache) { + if (cacheItem.db && cacheItem.nodes.includes(node)) { + cacheItem.nodes = cacheItem.nodes.filter((source) => source !== node); + } + } +}; + +/** + * Requests the mempool of trusted nodes with BIP35 `mempool` messages. + * + * Nodes don't announce transactions which a re-organization returns to their + * mempool, and the agent otherwise only hears about mempool transactions from + * unsolicited announcements. To keep `node_transaction` current, the node's + * mempool is requested: + * - for every connected node once mempool tracking is enabled, + * - when a node (re)connects after mempool tracking is enabled, and + * - after a re-organization reported by the node has been handled. + * + * The node answers with `inv` messages, which are handled by the existing + * announcement path. BIP35 replies may be split across many `inv` messages + * and have no completion marker, so a reply can only ever add + * `node_transaction` rows – a missing hash never proves absence. + * + * Requests are rate-limited per node: at most one request is sent per + * `minimumIntervalMs`, and requests during that window are coalesced into one + * request sent at the end of the window (so a re-organization shortly after a + * request is never lost, and a flapping node can't cause a request storm). + */ +export class MempoolResync { + private readonly options: Required>; + + private trackingEnabled = false; + + private stopped = false; + + private readonly lastRequestAt = new Map(); + + private readonly deferredRequests = new Map< + string, + { timer: unknown; triggers: Set } + >(); + + constructor(options: MempoolResyncOptions) { + this.options = { + clearTimer: (timer) => { + clearTimeout(timer as ReturnType); + }, + now: Date.now, + onRequestSent: () => undefined, + setTimer: (callback, ms) => setTimeout(callback, ms), + ...options, + }; + } + + /** + * Call when mempool tracking is enabled: requests the mempool of each given + * node, and enables requests on later connections and re-organizations. + */ + enableMempoolTracking(nodeNames: string[]) { + this.trackingEnabled = true; + nodeNames.forEach((nodeName) => { + this.request(nodeName, 'mempool-tracking-enabled'); + }); + } + + /** + * Call after a node has (re)connected and been registered. + */ + handleConnect(nodeName: string) { + this.request(nodeName, 'connect'); + } + + /** + * Call after a re-organization reported by the node has been handled. + */ + handleReorganization(nodeName: string) { + this.request(nodeName, 'reorganization'); + } + + /** + * Cancel any deferred requests and ignore all future triggers. + */ + stop() { + this.stopped = true; + this.deferredRequests.forEach(({ timer }) => { + this.options.clearTimer(timer); + }); + this.deferredRequests.clear(); + } + + private isActive() { + return this.options.enabled && this.trackingEnabled && !this.stopped; + } + + private millisecondsUntilNextRequestAllowed(nodeName: string) { + const lastRequestAt = this.lastRequestAt.get(nodeName); + if (lastRequestAt === undefined) { + return 0; + } + return Math.max( + 0, + lastRequestAt + this.options.minimumIntervalMs - this.options.now() + ); + } + + private request(nodeName: string, trigger: MempoolResyncTrigger) { + if (!this.isActive()) { + return; + } + const deferred = this.deferredRequests.get(nodeName); + if (deferred !== undefined) { + deferred.triggers.add(trigger); + return; + } + const delay = this.millisecondsUntilNextRequestAllowed(nodeName); + if (delay === 0) { + this.send(nodeName, [trigger]); + return; + } + const triggers = new Set([trigger]); + const timer = this.options.setTimer(() => { + this.deferredRequests.delete(nodeName); + this.send(nodeName, [...triggers]); + }, delay); + this.deferredRequests.set(nodeName, { timer, triggers }); + } + + private send(nodeName: string, triggers: MempoolResyncTrigger[]) { + if (this.stopped) { + return; + } + const node = this.options.getReadyNode(nodeName); + if (node === undefined) { + /** + * The node is disconnected: it will be requested again on reconnect. + */ + return; + } + forgetSavedAcknowledgementsFromNode(node, this.options.transactionCache); + node.peer.sendMessage(new node.peer.messages.MemPool()); + this.lastRequestAt.set(nodeName, this.options.now()); + this.options.onRequestSent(nodeName, triggers); + } +} diff --git a/src/components/pg-binary-copy.e2e.spec.ts b/src/components/pg-binary-copy.e2e.spec.ts new file mode 100644 index 00000000..0c5b2727 --- /dev/null +++ b/src/components/pg-binary-copy.e2e.spec.ts @@ -0,0 +1,80 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import type { TestFn } from 'ava'; +import baseTest from 'ava'; +import pg from 'pg'; + +import { BinaryCopyWriter, copyFromBuffers } from './pg-binary-copy.js'; + +const test = baseTest as TestFn<{ client: pg.Client }>; +const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; +const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; + +/* + * Runs against a real server, using only temporary tables (no persistent + * tables are created or modified). + */ +test.before(async (t) => { + t.context.client = new pg.Client({ + connectionString: `postgres://chaingraph:very_insecure_postgres_password@${host}:${port}/postgres`, + }); + await t.context.client.connect(); + await t.context.client.query(/* sql */ ` +SET statement_timeout = '5s'; +CREATE TEMP TABLE stage_a (id bigint NOT NULL); +CREATE TEMP TABLE stage_b (id bigint NOT NULL); +`); +}); + +test.after.always(async (t) => { + await t.context.client.end(); +}); + +const rows = (values: (number | null)[]) => + values + .reduce( + (writer, value) => + value === null + ? writer.startRow(1).null() + : writer.startRow(1).int8(value), + new BinaryCopyWriter() + ) + .finish(); +const copyA = 'COPY pg_temp.stage_a (id) FROM STDIN WITH (FORMAT binary)'; +const copyB = 'COPY pg_temp.stage_b (id) FROM STDIN WITH (FORMAT binary)'; +const ids = async (client: pg.Client, table: string) => + ( + await client.query<{ id: string }>( + `SELECT id FROM pg_temp.${table} ORDER BY id;` + ) + ).rows.map((row) => Number(row.id)); + +test.serial( + '[e2e] copyFromBuffers: a failed later COPY rolls back and the connection stays usable', + async (t) => { + const { client } = t.context; + await client.query('BEGIN;'); + const error = await t.throwsAsync( + copyFromBuffers(client, [ + { messages: rows([1, 2]), statement: copyA }, + { messages: rows([3, null]), statement: copyB }, + { messages: rows([5]), statement: copyA }, + ]) + ); + t.regex(error?.message ?? '', /null value/u); + await client.query('ROLLBACK;'); + t.deepEqual(await ids(client, 'stage_a'), []); + t.deepEqual(await ids(client, 'stage_b'), []); + + await client.query('BEGIN;'); + t.deepEqual( + await copyFromBuffers(client, [ + { messages: rows([10, 11]), statement: copyA }, + { messages: rows([12]), statement: copyB }, + ]), + [2, 1] + ); + await client.query('COMMIT;'); + t.deepEqual(await ids(client, 'stage_a'), [10, 11]); + t.deepEqual(await ids(client, 'stage_b'), [12]); + } +); diff --git a/src/components/pg-binary-copy.spec.ts b/src/components/pg-binary-copy.spec.ts new file mode 100644 index 00000000..3110f2e2 --- /dev/null +++ b/src/components/pg-binary-copy.spec.ts @@ -0,0 +1,244 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import test from 'ava'; + +import { + binaryCopySignature, + BinaryCopyWriter, + copyDataPayload, + CopyFromBuffersQuery, + decodeBinaryCopy, +} from './pg-binary-copy.js'; + +const header = Buffer.concat([ + binaryCopySignature, + Buffer.from('0000000000000000', 'hex'), +]); +const trailer = Buffer.from('ffff', 'hex'); + +const copyDone = Buffer.from('6300000004', 'hex'); +const copyData = (payload: Buffer) => { + const messageHeader = Buffer.alloc(5); + messageHeader[0] = 0x64; + messageHeader.writeInt32BE(4 + payload.length, 1); + return Buffer.concat([messageHeader, payload, copyDone]); +}; + +test('BinaryCopyWriter: empty payload is header + trailer, framed as CopyData + CopyDone', (t) => { + const messages = new BinaryCopyWriter().finish(); + t.deepEqual(messages, copyData(Buffer.concat([header, trailer]))); + t.deepEqual(copyDataPayload(messages), Buffer.concat([header, trailer])); + t.deepEqual(decodeBinaryCopy(messages), []); +}); + +test('copyDataPayload: rejects anything but one CopyData + CopyDone', (t) => { + const messages = new BinaryCopyWriter().finish(); + t.throws(() => copyDataPayload(messages.subarray(0, messages.length - 1))); + t.throws(() => copyDataPayload(Buffer.concat([messages, copyDone]))); + const wrongCode = Buffer.from(messages); + wrongCode[0] = 0x51; + t.throws(() => copyDataPayload(wrongCode)); +}); + +test('BinaryCopyWriter: exact wire bytes for every field type', (t) => { + const payload = new BinaryCopyWriter() + .startRow(6) + .hexBytea('c0de') + .int8(1) + .int8(-2n) + .boolean(true) + .text('minting') + .null() + .finish(); + t.deepEqual( + copyDataPayload(payload), + Buffer.concat([ + header, + Buffer.from('0006', 'hex'), + Buffer.from('00000002c0de', 'hex'), + Buffer.from('000000080000000000000001', 'hex'), + Buffer.from('00000008fffffffffffffffe', 'hex'), + Buffer.from('0000000101', 'hex'), + Buffer.from('000000076d696e74696e67', 'hex'), + Buffer.from('ffffffff', 'hex'), + trailer, + ]) + ); +}); + +const int8Cases: [bigint | number, string][] = [ + [0, '0000000000000000'], + [1, '0000000000000001'], + [-1, 'ffffffffffffffff'], + [0xffffffff, '00000000ffffffff'], + [0x1_0000_0000, '0000000100000000'], + [-0x1_0000_0000, 'ffffffff00000000'], + // cspell:disable-next-line + [-0x1_0000_0001, 'fffffffeffffffff'], + [Number.MAX_SAFE_INTEGER, '001fffffffffffff'], + [Number.MIN_SAFE_INTEGER, 'ffe0000000000001'], + [9223372036854775807n, '7fffffffffffffff'], + [-9223372036854775808n, '8000000000000000'], + [2_100_000_000_000_000n, '000775f05a074000'], +]; + +test('BinaryCopyWriter: int8 round-trips (number and bigint)', (t) => { + const writer = new BinaryCopyWriter(1); + int8Cases.forEach(([value]) => writer.startRow(1).int8(value)); + const rows = decodeBinaryCopy(writer.finish()); + t.is(rows.length, int8Cases.length); + rows.forEach((row, index) => { + const [value, hex] = int8Cases[index]!; + t.is(row[0]!.toString('hex'), hex); + t.is(row[0]!.readBigInt64BE(0), BigInt(value)); + }); +}); + +test('BinaryCopyWriter: int8 rejects unsafe or fractional numbers', (t) => { + t.throws(() => new BinaryCopyWriter().int8(Number.MAX_SAFE_INTEGER + 1)); + t.throws(() => new BinaryCopyWriter().int8(1.5)); + t.throws(() => new BinaryCopyWriter().int8(NaN)); + t.throws(() => new BinaryCopyWriter().int8(2n ** 63n)); +}); + +test('BinaryCopyWriter: bytea round-trips (empty, hex, raw, large) and rejects bad hex', (t) => { + const large = Buffer.alloc(3 * 1024 * 1024, 0xab); + const rows = decodeBinaryCopy( + new BinaryCopyWriter(1) + .startRow(4) + .hexBytea('') + .hexBytea('00ff10') + .bytea(Uint8Array.from([1, 2, 3])) + .hexBytea(large.toString('hex')) + .finish() + ); + t.is(rows.length, 1); + t.deepEqual(rows[0]![0], Buffer.alloc(0)); + t.deepEqual(rows[0]![1], Buffer.from('00ff10', 'hex')); + t.deepEqual(rows[0]![2], Buffer.from([1, 2, 3])); + t.true(rows[0]![3]!.equals(large)); + t.throws(() => new BinaryCopyWriter().hexBytea('abc')); + t.throws(() => new BinaryCopyWriter().hexBytea('zz')); +}); + +test('BinaryCopyWriter: booleans, enum labels and nulls', (t) => { + const rows = decodeBinaryCopy( + new BinaryCopyWriter() + .startRow(5) + .boolean(false) + .boolean(true) + .text('none') + .text('mutable') + .null() + .finish() + ); + t.deepEqual(rows, [ + [ + Buffer.from([0]), + Buffer.from([1]), + Buffer.from('none'), + Buffer.from('mutable'), + null, + ], + ]); +}); + +test('BinaryCopyWriter: grows past its initial capacity', (t) => { + const writer = new BinaryCopyWriter(1); + const rowCount = 10_000; + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let index = 0; index < rowCount; index += 1) { + writer.startRow(2).int8(index).hexBytea('0102'); + } + const rows = decodeBinaryCopy(writer.finish()); + t.is(writer.rowCount, rowCount); + t.is(rows.length, rowCount); + t.is(rows[rowCount - 1]![0]!.readBigInt64BE(0), BigInt(rowCount - 1)); +}); + +const parseFrontendMessages = (bytes: Buffer) => { + const messages: { code: string; body: Buffer }[] = []; + // eslint-disable-next-line functional/no-let + let offset = 0; + // eslint-disable-next-line functional/no-loop-statement + while (offset < bytes.length) { + const code = String.fromCharCode(bytes[offset]!); + const length = bytes.readInt32BE(offset + 1); + messages.push({ + body: bytes.subarray(offset + 5, offset + 1 + length), + code, + }); + offset += 1 + length; + } + return messages; +}; + +const mockConnection = () => { + const writes: Buffer[] = []; + const events: string[] = []; + const connection = { + query: (text: string) => { + const body = Buffer.from(`${text}\0`); + const queryHeader = Buffer.alloc(5); + queryHeader[0] = 'Q'.charCodeAt(0); + queryHeader.writeInt32BE(4 + body.length, 1); + writes.push(queryHeader, body); + }, + stream: { + cork: () => events.push('cork'), + uncork: () => events.push('uncork'), + write: (chunk: Buffer) => { + writes.push(Buffer.from(chunk)); + return true; + }, + }, + }; + return { connection, events, writes }; +}; + +test('CopyFromBuffersQuery: one corked write of Query + CopyData/CopyDone per statement', (t) => { + const first = new BinaryCopyWriter().startRow(1).int8(7).finish(); + const second = new BinaryCopyWriter().finish(); + const { connection, events, writes } = mockConnection(); + const query = new CopyFromBuffersQuery([ + { messages: first, statement: 'COPY a FROM STDIN' }, + { messages: second, statement: 'COPY b FROM STDIN' }, + ]); + const results: [Error | undefined, (number | null)[]][] = []; + query.callback = (err, rowCounts) => results.push([err, rowCounts]); + query.submit(connection as never); + t.deepEqual(events, ['cork', 'uncork']); + const messages = parseFrontendMessages(Buffer.concat(writes)); + t.deepEqual( + messages.map((message) => message.code), + ['Q', 'd', 'c', 'd', 'c'] + ); + t.is( + messages[0]!.body.toString(), + 'COPY a FROM STDIN;\nCOPY b FROM STDIN;\0' + ); + t.deepEqual(messages[1]!.body, copyDataPayload(first)); + t.deepEqual(messages[3]!.body, copyDataPayload(second)); + t.deepEqual(messages[2]!.body.length, 0); + query.handleCopyInResponse(); + query.handleCommandComplete({ text: 'COPY 1' }); + query.handleCopyInResponse(); + query.handleCommandComplete({ text: 'COPY 0' }); + query.handleReadyForQuery(); + query.handleReadyForQuery(); + t.deepEqual(results, [[undefined, [1, 0]]]); +}); + +test('CopyFromBuffersQuery: errors reach the callback once', (t) => { + const query = new CopyFromBuffersQuery([ + { + messages: new BinaryCopyWriter().finish(), + statement: 'COPY x FROM STDIN', + }, + ]); + const results: (Error | undefined)[] = []; + query.callback = (err) => results.push(err); + const error = new Error('boom'); + query.handleError(error); + query.handleReadyForQuery(); + t.deepEqual(results, [error]); +}); diff --git a/src/components/pg-binary-copy.ts b/src/components/pg-binary-copy.ts new file mode 100644 index 00000000..3a1a6770 --- /dev/null +++ b/src/components/pg-binary-copy.ts @@ -0,0 +1,434 @@ +/* eslint-disable max-classes-per-file */ +/** + * Minimal support for Postgres `COPY … FROM STDIN (FORMAT binary)` on top of + * the `pg` driver, without additional dependencies. + * + * - `BinaryCopyWriter` encodes rows in the binary COPY file format + * (https://www.postgresql.org/docs/current/sql-copy.html#id-1.9.3.55.9.4) + * directly inside the frontend protocol messages which carry them (one + * CopyData message followed by CopyDone), so they can be written to the + * socket without another copy. + * - `CopyFromBuffersQuery` is a `pg` "submittable" which runs one or more + * `COPY … FROM STDIN` statements and pipelines their encoded data (the same + * protocol hooks `pg-copy-streams` uses). + */ +import type pg from 'pg'; + +/** + * `PGCOPY\n\377\r\n\0`, followed by a 32-bit flags field (0) and a 32-bit + * header extension length (0). + */ +export const binaryCopySignature = Buffer.from([ + /* eslint-disable @typescript-eslint/no-magic-numbers */ + 0x50, 0x47, 0x43, 0x4f, 0x50, 0x59, 0x0a, 0xff, 0x0d, 0x0a, 0x00, + /* eslint-enable @typescript-eslint/no-magic-numbers */ +]); +const flagsAndExtensionBytes = 8; +const headerBytes = binaryCopySignature.length + flagsAndExtensionBytes; +/** A frontend message's type byte plus its int32 length. */ +const messageHeaderBytes = 5; +const messageLengthBytes = 4; +/** CopyData header before the data, CopyDone after it. */ +const framingBytes = messageHeaderBytes + messageHeaderBytes; +/** `d` */ +const copyDataCode = 0x64; +/** `c` */ +const copyDoneCode = 0x63; +/** + * Postgres rejects messages over 1 GB (`PQ_LARGE_MESSAGE_LIMIT`); even a + * 32 MB block encodes to well under 100 MB per table. + */ +const maxCopyDataBytes = 0x3fff_fff0; +const fieldCountBytes = 2; +const fieldLengthBytes = 4; +const int8Bytes = 8; +const booleanBytes = 1; +const nullFieldLength = -1; +const trailer = -1; +const uint32Range = 0x1_0000_0000; +const hexCharsPerByte = 2; +const maxErrorHexChars = 64; +const bytesPerKilobyte = 1024; +const initialCapacityKilobytes = 64; +const initialCapacity = initialCapacityKilobytes * bytesPerKilobyte; +const growthFactor = 2; + +/** + * Encode rows in the Postgres binary COPY format. Fields must be written in + * the column order of the `COPY` statement. Integer types are written as + * `int8`, so every integer column must be `bigint`; `enum` values are sent as + * their label (that is the binary format of `enum_send`/`enum_recv`). + * + * The data is written after a reserved CopyData message header, and `finish` + * appends a CopyDone message: the result is ready to send as-is after a + * `COPY … FROM STDIN` Query message. + */ +export class BinaryCopyWriter { + buffer: Buffer; + + offset: number; + + rowCount = 0; + + constructor(expectedSizeBytes = initialCapacity) { + this.buffer = Buffer.allocUnsafe( + Math.max( + expectedSizeBytes + framingBytes, + framingBytes + headerBytes + fieldCountBytes + ) + ); + const start = messageHeaderBytes; + binaryCopySignature.copy(this.buffer, start); + this.buffer.writeInt32BE(0, start + binaryCopySignature.length); + this.buffer.writeInt32BE( + 0, + start + binaryCopySignature.length + fieldLengthBytes + ); + this.offset = start + headerBytes; + } + + ensure(additionalBytes: number) { + const required = this.offset + additionalBytes; + if (required <= this.buffer.length) { + return; + } + const grown = Buffer.allocUnsafe( + Math.max(required, this.buffer.length * growthFactor) + ); + this.buffer.copy(grown, 0, 0, this.offset); + this.buffer = grown; + } + + startRow(fieldCount: number) { + this.ensure(fieldCountBytes); + this.buffer.writeInt16BE(fieldCount, this.offset); + this.offset += fieldCountBytes; + this.rowCount += 1; + return this; + } + + null() { + this.ensure(fieldLengthBytes); + this.buffer.writeInt32BE(nullFieldLength, this.offset); + this.offset += fieldLengthBytes; + return this; + } + + /** + * Write a `bigint` (int8) field. Numbers must be safe integers. + */ + int8(value: bigint | number) { + this.ensure(fieldLengthBytes + int8Bytes); + this.buffer.writeInt32BE(int8Bytes, this.offset); + this.offset += fieldLengthBytes; + if (typeof value === 'bigint') { + this.buffer.writeBigInt64BE(value, this.offset); + } else { + if (!Number.isSafeInteger(value)) { + // eslint-disable-next-line functional/no-throw-statement + throw new RangeError(`Cannot encode ${value} as int8.`); + } + const high = Math.floor(value / uint32Range); + const low = value - high * uint32Range; + this.buffer.writeInt32BE(high, this.offset); + this.buffer.writeUInt32BE(low, this.offset + fieldLengthBytes); + } + this.offset += int8Bytes; + return this; + } + + boolean(value: boolean) { + this.ensure(fieldLengthBytes + booleanBytes); + this.buffer.writeInt32BE(booleanBytes, this.offset); + this.offset += fieldLengthBytes; + this.buffer[this.offset] = value ? 1 : 0; + this.offset += booleanBytes; + return this; + } + + /** + * Write a `bytea` field from a hex string (without `\x` prefix). + */ + hexBytea(hex: string) { + if (hex.length % hexCharsPerByte !== 0) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Invalid hex (odd length): ${hex.slice(0, maxErrorHexChars)}` + ); + } + const byteLength = hex.length / hexCharsPerByte; + this.ensure(fieldLengthBytes + byteLength); + this.buffer.writeInt32BE(byteLength, this.offset); + this.offset += fieldLengthBytes; + const written = this.buffer.write(hex, this.offset, byteLength, 'hex'); + if (written !== byteLength) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error(`Invalid hex: ${hex.slice(0, maxErrorHexChars)}`); + } + this.offset += byteLength; + return this; + } + + bytea(bytes: Uint8Array) { + this.ensure(fieldLengthBytes + bytes.length); + this.buffer.writeInt32BE(bytes.length, this.offset); + this.offset += fieldLengthBytes; + this.buffer.set(bytes, this.offset); + this.offset += bytes.length; + return this; + } + + /** + * Write a `text` (or `enum` label) field. + */ + text(value: string) { + const byteLength = Buffer.byteLength(value, 'utf8'); + this.ensure(fieldLengthBytes + byteLength); + this.buffer.writeInt32BE(byteLength, this.offset); + this.offset += fieldLengthBytes; + this.buffer.write(value, this.offset, byteLength, 'utf8'); + this.offset += byteLength; + return this; + } + + /** + * Append the trailer and return the frontend messages carrying the data: a + * CopyData message with the complete binary COPY payload, then CopyDone (a + * view of the internal buffer). The writer must not be used afterwards. + */ + finish() { + this.ensure(fieldCountBytes + messageHeaderBytes); + this.buffer.writeInt16BE(trailer, this.offset); + this.offset += fieldCountBytes; + const payloadBytes = this.offset - messageHeaderBytes; + if (payloadBytes > maxCopyDataBytes) { + // eslint-disable-next-line functional/no-throw-statement + throw new RangeError( + `COPY payload of ${payloadBytes} bytes exceeds the CopyData message limit.` + ); + } + this.buffer[0] = copyDataCode; + this.buffer.writeInt32BE(messageLengthBytes + payloadBytes, 1); + this.buffer[this.offset] = copyDoneCode; + this.buffer.writeInt32BE(messageLengthBytes, this.offset + 1); + this.offset += messageHeaderBytes; + return this.buffer.subarray(0, this.offset); + } +} + +const decodeRow = (payload: Buffer, start: number, fieldCount: number) => { + const row: (Buffer | null)[] = []; + // eslint-disable-next-line functional/no-let + let offset = start; + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let field = 0; field < fieldCount; field += 1) { + const length = payload.readInt32BE(offset); + offset += fieldLengthBytes; + const isNull = length === nullFieldLength; + row.push(isNull ? null : payload.subarray(offset, offset + length)); + offset += isNull ? 0 : length; + } + return { offset, row }; +}; + +/** + * Extract the binary COPY payload from the messages built by + * `BinaryCopyWriter.finish` (one CopyData message, then CopyDone). + */ +export const copyDataPayload = (messages: Buffer) => { + const copyDataLength = messages.readInt32BE(1); + const copyDoneOffset = 1 + copyDataLength; + if ( + messages[0] !== copyDataCode || + messages[copyDoneOffset] !== copyDoneCode || + messages.readInt32BE(copyDoneOffset + 1) !== messageLengthBytes || + copyDoneOffset + messageHeaderBytes !== messages.length + ) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error('Expected exactly one CopyData message and CopyDone.'); + } + return messages.subarray(messageHeaderBytes, copyDoneOffset); +}; + +/** + * Decode the messages built by `BinaryCopyWriter.finish`. Only used to verify + * encoders: every non-null field is returned as raw bytes. + */ +export const decodeBinaryCopy = (messages: Buffer) => { + const payload = copyDataPayload(messages); + if ( + !payload.subarray(0, binaryCopySignature.length).equals(binaryCopySignature) + ) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error('Missing binary COPY signature.'); + } + const extensionLength = payload.readInt32BE( + binaryCopySignature.length + fieldLengthBytes + ); + const rows: (Buffer | null)[][] = []; + // eslint-disable-next-line functional/no-let + let offset = headerBytes + extensionLength; + // eslint-disable-next-line functional/no-let + let fieldCount = payload.readInt16BE(offset); + // eslint-disable-next-line functional/no-loop-statement + while (fieldCount !== trailer) { + const { offset: rowEnd, row } = decodeRow( + payload, + offset + fieldCountBytes, + fieldCount + ); + rows.push(row); + offset = rowEnd; + fieldCount = payload.readInt16BE(offset); + } + if (offset + fieldCountBytes !== payload.length) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error('Trailing bytes after binary COPY trailer.'); + } + return rows; +}; + +// eslint-disable-next-line functional/no-mixed-type +interface CopyConnection { + query: (text: string) => void; + stream: { + cork: () => void; + uncork: () => void; + write: (chunk: Buffer) => boolean; + }; +} + +export interface CopyFromStdin { + /** + * A `COPY … FROM STDIN` statement (without trailing semicolon). + */ + statement: string; + /** + * The CopyData and CopyDone messages for this statement, from + * `BinaryCopyWriter.finish`. + */ + messages: Buffer; +} + +/** + * A `pg` submittable which runs one or more `COPY … FROM STDIN` statements in + * a single simple-protocol Query message and pipelines all of their data + * right behind it (Query, then CopyData + CopyDone per statement, in one + * corked socket write). This costs one round trip regardless of the number of + * statements, instead of one round trip per statement plus one per + * CopyInResponse. + * + * Pipelining is safe: if a statement fails, Postgres skips the rest of the + * Query message, sends ReadyForQuery and drops the remaining CopyData and + * CopyDone messages ("Copy-In Mode" in the protocol documentation). + * + * Use via `copyFromBuffers`. + */ +export class CopyFromBuffersQuery implements pg.Submittable { + copies: CopyFromStdin[]; + + rowCounts: (number | null)[] = []; + + error: Error | undefined; + + done = false; + + callback: + | ((err: Error | undefined, rowCounts: (number | null)[]) => void) + | undefined; + + constructor(copies: CopyFromStdin[]) { + this.copies = copies; + } + + finish(err: Error | undefined) { + if (this.done) { + return; + } + this.done = true; + this.callback?.(err, this.rowCounts); + } + + submit(connection: pg.Connection) { + const copyConnection = connection as unknown as CopyConnection; + const { stream } = copyConnection; + stream.cork(); + copyConnection.query( + this.copies.map((copy) => `${copy.statement};`).join('\n') + ); + this.copies.forEach(({ messages }) => { + stream.write(messages); + }); + stream.uncork(); + } + + /** + * The data was already sent in `submit`. + */ + // eslint-disable-next-line class-methods-use-this + handleCopyInResponse() { + /* already sent */ + } + + handleCommandComplete(message: { text?: string }) { + const match = /^COPY (?\d+)$/u.exec(message.text ?? ''); + this.rowCounts.push( + match?.groups?.count === undefined ? null : Number(match.groups.count) + ); + } + + handleReadyForQuery() { + this.finish(this.error); + } + + handleError(err: Error) { + this.error = err; + this.finish(err); + } + + /* The remaining hooks are required by `pg` but unused by COPY FROM. */ + // eslint-disable-next-line class-methods-use-this + handleRowDescription() { + /* unused */ + } + + // eslint-disable-next-line class-methods-use-this + handleDataRow() { + /* unused */ + } + + // eslint-disable-next-line class-methods-use-this + handleEmptyQuery() { + /* unused */ + } + + // eslint-disable-next-line class-methods-use-this + handlePortalSuspended() { + /* unused */ + } + + // eslint-disable-next-line class-methods-use-this + handleCopyData() { + /* unused */ + } +} + +/** + * Run `COPY … FROM STDIN` statements on `client` with already-encoded data, + * in one round trip. Resolves to the row count reported for each statement. + */ +export const copyFromBuffers = async ( + client: pg.ClientBase, + copies: CopyFromStdin[] +) => + new Promise<(number | null)[]>((resolve, reject) => { + const query = new CopyFromBuffersQuery(copies); + query.callback = (err, rowCounts) => { + if (err === undefined) { + resolve(rowCounts); + } else { + reject(err); + } + }; + client.query(query); + }); diff --git a/src/components/sync-state.spec.ts b/src/components/sync-state.spec.ts index b21a1f54..0b6e5b45 100644 --- a/src/components/sync-state.spec.ts +++ b/src/components/sync-state.spec.ts @@ -58,3 +58,67 @@ test('SyncState', (t) => { t.deepEqual(state.additionalSyncedHeights, []); t.deepEqual(state.latestSyncedBlockTime, new Date(5)); }); + +test('SyncState scales when marking many contiguous heights as pending', (t) => { + const state = new SyncState({ + additionalSyncedHeights: [], + fullySyncedUpToHeight: 0, + pendingSyncOfHeights: [], + }); + const pendingThroughHeight = 4_000; + const maximumDurationMs = 1_000; + + const startedAt = Date.now(); + // eslint-disable-next-line functional/no-loop-statement + for ( + // eslint-disable-next-line functional/no-let + let nextHeight = state.getPendingSyncHeight() + 1; + nextHeight <= pendingThroughHeight; + nextHeight = state.getPendingSyncHeight() + 1 + ) { + state.markHeightAsPendingSync(nextHeight); + } + const durationMs = Date.now() - startedAt; + + t.deepEqual(state.getPendingSyncHeight(), pendingThroughHeight); + t.true( + durationMs < maximumDurationMs, + `Expected marking contiguous pending heights to complete in less than ${maximumDurationMs}ms; took ${durationMs}ms.` + ); +}); + +test('SyncState scales when marking many pending heights as synced', (t) => { + const state = new SyncState({ + additionalSyncedHeights: [], + fullySyncedUpToHeight: 0, + pendingSyncOfHeights: [], + }); + /* + * Mirrors `catchUpViaHeaders` accepting a long run of already-known blocks + * (e.g. a new network sharing history with an existing one): every height is + * marked pending, then every height is marked synced in ascending order. With + * array-backed state this was quadratic (~7s at 300k heights). + */ + const acceptedThroughHeight = 300_000; + const maximumDurationMs = 1_000; + + const startedAt = Date.now(); + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let height = 1; height <= acceptedThroughHeight; height += 1) { + state.markHeightAsPendingSync(height); + } + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let height = 1; height <= acceptedThroughHeight; height += 1) { + state.markHeightAsSynced(height, 'caught-up'); + } + const durationMs = Date.now() - startedAt; + + t.deepEqual(state.fullySyncedUpToHeight, acceptedThroughHeight); + t.deepEqual(state.getPendingSyncHeight(), acceptedThroughHeight); + t.deepEqual(state.pendingSyncOfHeights, []); + t.deepEqual(state.additionalSyncedHeights, []); + t.true( + durationMs < maximumDurationMs, + `Expected marking pending heights as synced to complete in less than ${maximumDurationMs}ms; took ${durationMs}ms.` + ); +}); diff --git a/src/components/sync-state.ts b/src/components/sync-state.ts index ed15b605..6a40f660 100644 --- a/src/components/sync-state.ts +++ b/src/components/sync-state.ts @@ -4,15 +4,6 @@ export interface InitialSyncState { additionalSyncedHeights: number[]; } -const removeValueIfPresent = (array: number[], value: number) => { - const index = array.indexOf(value); - if (index !== -1) { - array.splice(index, 1); - return true; - } - return false; -}; - /** * A simple data structure to keep track of the known database syncing progress * for a particular node. @@ -20,10 +11,6 @@ const removeValueIfPresent = (array: number[], value: number) => { export class SyncState { fullySyncedUpToHeight: number; - pendingSyncOfHeights: number[]; - - additionalSyncedHeights: number[]; - /** * The block time claimed by the latest synced block. Chaingraph assumes the * trusted node is itself not fully-synced if this time is older than 2 hours @@ -48,14 +35,43 @@ export class SyncState { */ latestSyncedBlockTime: Date | 'caught-up' | undefined; + private pendingSyncHeight: number; + + private pendingSyncHeights: Set; + + private additionalSyncedHeightSet: Set; + constructor(initialState: InitialSyncState) { this.fullySyncedUpToHeight = initialState.fullySyncedUpToHeight; - this.pendingSyncOfHeights = initialState.pendingSyncOfHeights.slice(); - this.additionalSyncedHeights = initialState.additionalSyncedHeights.slice(); + this.pendingSyncHeights = new Set(initialState.pendingSyncOfHeights); + this.additionalSyncedHeightSet = new Set( + initialState.additionalSyncedHeights + ); + this.pendingSyncHeight = this.computePendingSyncHeightFrom( + this.fullySyncedUpToHeight + ); this.latestSyncedBlockTime = initialState.fullySyncedUpToHeight > 0 ? 'caught-up' : undefined; } + /** + * Heights currently pending sync, in the order they were marked. (Derived + * from a `Set`: removing a height is O(1), so marking many heights as synced + * – e.g. after `catchUpViaHeaders` accepts a long run of known blocks – is + * linear rather than quadratic.) + */ + get pendingSyncOfHeights() { + return [...this.pendingSyncHeights]; + } + + /** + * Heights above `fullySyncedUpToHeight` which have been synced, in the order + * they were marked. + */ + get additionalSyncedHeights() { + return [...this.additionalSyncedHeightSet]; + } + // eslint-disable-next-line complexity markHeightAsSynced(height: number, claimedBlockTime: Date | 'caught-up') { if (claimedBlockTime === 'caught-up') { @@ -69,28 +85,30 @@ export class SyncState { if ( this.fullySyncedUpToHeight < height && - !this.additionalSyncedHeights.includes(height) + !this.additionalSyncedHeightSet.has(height) ) { - this.additionalSyncedHeights.push(height); - removeValueIfPresent(this.pendingSyncOfHeights, height); + this.additionalSyncedHeightSet.add(height); + this.pendingSyncHeights.delete(height); } // eslint-disable-next-line functional/no-let let nextHeight = this.fullySyncedUpToHeight + 1; // eslint-disable-next-line functional/no-loop-statement - while (removeValueIfPresent(this.additionalSyncedHeights, nextHeight)) { + while (this.additionalSyncedHeightSet.delete(nextHeight)) { this.fullySyncedUpToHeight = nextHeight; nextHeight += 1; } + this.updatePendingSyncHeight(); } markHeightAsPendingSync(height: number) { if ( this.fullySyncedUpToHeight < height && - !this.pendingSyncOfHeights.includes(height) && - !this.additionalSyncedHeights.includes(height) + !this.pendingSyncHeights.has(height) && + !this.additionalSyncedHeightSet.has(height) ) { - this.pendingSyncOfHeights.push(height); + this.pendingSyncHeights.add(height); + this.updatePendingSyncHeight(); } } @@ -107,12 +125,13 @@ export class SyncState { ? this.fullySyncedUpToHeight : height - 1; - this.pendingSyncOfHeights = this.pendingSyncOfHeights.filter( - (pending) => pending < height + this.pendingSyncHeights = new Set( + this.pendingSyncOfHeights.filter((pending) => pending < height) ); - this.additionalSyncedHeights = this.additionalSyncedHeights.filter( - (completed) => completed < height + this.additionalSyncedHeightSet = new Set( + this.additionalSyncedHeights.filter((completed) => completed < height) ); + this.resetPendingSyncHeight(); } /** @@ -120,16 +139,35 @@ export class SyncState { * useful for prioritizing syncing. */ getPendingSyncHeight() { - const handledHeights = [ - ...this.pendingSyncOfHeights, - ...this.additionalSyncedHeights, - ]; + return this.pendingSyncHeight; + } + + private isHeightHandled(height: number) { + return ( + this.pendingSyncHeights.has(height) || + this.additionalSyncedHeightSet.has(height) + ); + } + + private computePendingSyncHeightFrom(height: number) { // eslint-disable-next-line functional/no-let - let firstUnhandledHeight = this.fullySyncedUpToHeight + 1; + let pendingSyncHeight = height; // eslint-disable-next-line functional/no-loop-statement - while (handledHeights.includes(firstUnhandledHeight)) { - firstUnhandledHeight += 1; + while (this.isHeightHandled(pendingSyncHeight + 1)) { + pendingSyncHeight += 1; } - return firstUnhandledHeight - 1; + return pendingSyncHeight; + } + + private updatePendingSyncHeight() { + this.pendingSyncHeight = this.computePendingSyncHeightFrom( + Math.max(this.pendingSyncHeight, this.fullySyncedUpToHeight) + ); + } + + private resetPendingSyncHeight() { + this.pendingSyncHeight = this.computePendingSyncHeightFrom( + this.fullySyncedUpToHeight + ); } } diff --git a/src/config.ts b/src/config.ts index 46f51ff1..e19eff5d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,16 +36,22 @@ const configuration = { const expectedOptions = [ 'CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB', 'CHAINGRAPH_GENESIS_BLOCKS', + 'CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE', 'CHAINGRAPH_INTERNAL_API_PORT', 'CHAINGRAPH_LOG_FIREHOSE', 'CHAINGRAPH_LOG_LEVEL_STDOUT', 'CHAINGRAPH_LOG_LEVEL_PATH', 'CHAINGRAPH_LOG_PATH', + 'CHAINGRAPH_MEMPOOL_RESYNC', + 'CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS', + 'CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS', + 'CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS', 'CHAINGRAPH_POSTGRES_CONNECTION_STRING', 'CHAINGRAPH_POSTGRES_MAX_CONNECTIONS', 'CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT', 'CHAINGRAPH_TRUSTED_NODES', 'CHAINGRAPH_USER_AGENT', + 'CHAINGRAPH_WRITE_PATH', 'NODE_ENV', ] as const; const requireStringValues = ( @@ -112,6 +118,73 @@ if (isNaN(blockBufferTargetSizeMb) || blockBufferTargetSizeMb <= 0) { ); } +const incompleteBlockRepairBatchSize = Number( + configuration.CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE +); +if ( + !Number.isInteger(incompleteBlockRepairBatchSize) || + incompleteBlockRepairBatchSize < 0 +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + 'The CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE environment variable must be an integer greater than or equal to 0.' + ); +} + +const mempoolTransactionExpirationMs = Number( + configuration.CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS +); +if ( + !Number.isInteger(mempoolTransactionExpirationMs) || + mempoolTransactionExpirationMs <= 0 +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + 'The CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_MS environment variable must be an integer greater than 0.' + ); +} + +const mempoolTransactionExpirationScanIntervalMs = Number( + configuration.CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS +); +if ( + !Number.isInteger(mempoolTransactionExpirationScanIntervalMs) || + mempoolTransactionExpirationScanIntervalMs <= 0 +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + 'The CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS environment variable must be an integer greater than 0.' + ); +} + +if ( + configuration.CHAINGRAPH_MEMPOOL_RESYNC.toLowerCase() !== 'true' && + configuration.CHAINGRAPH_MEMPOOL_RESYNC.toLowerCase() !== 'false' +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Invalid value provided in the 'CHAINGRAPH_MEMPOOL_RESYNC' environment variable: ${configuration.CHAINGRAPH_MEMPOOL_RESYNC}. Must be one of the following: true, false` + ); +} +/** + * Set via the `CHAINGRAPH_MEMPOOL_RESYNC` environment variable. + */ +const mempoolResync = + configuration.CHAINGRAPH_MEMPOOL_RESYNC.toLowerCase() === 'true'; + +const mempoolResyncMinIntervalMs = Number( + configuration.CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS +); +if ( + !Number.isInteger(mempoolResyncMinIntervalMs) || + mempoolResyncMinIntervalMs < 0 +) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + 'The CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS environment variable must be an integer greater than or equal to 0.' + ); +} + const extendTildeAndResolvePath = (path: string) => path.startsWith('~') ? resolve(join(homedir(), path.slice(1))) @@ -367,6 +440,24 @@ const postgresSynchronousCommit = */ const isProduction = configuration.NODE_ENV === 'production'; +const writePaths = ['sql', 'copy'] as const; +const isWritePath = (value: string): value is (typeof writePaths)[number] => + writePaths.includes(value as (typeof writePaths)[number]); +if (!isWritePath(configuration.CHAINGRAPH_WRITE_PATH)) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Invalid value provided in the 'CHAINGRAPH_WRITE_PATH' environment variable: ${ + configuration.CHAINGRAPH_WRITE_PATH + }. Must be one of the following: ${writePaths.join(', ')}` + ); +} +/** + * Set via the `CHAINGRAPH_WRITE_PATH` environment variable: `sql` (multi-row + * `INSERT … VALUES` statements) or `copy` (binary `COPY` into temporary + * staging tables, then `INSERT … SELECT`). + */ +const chaingraphWritePath = configuration.CHAINGRAPH_WRITE_PATH; + export { blockBufferTargetSizeMb, chaingraphInternalApiPort, @@ -375,7 +466,13 @@ export { chaingraphLogLevelStdout, chaingraphLogLevelPath, chaingraphUserAgent, + chaingraphWritePath, genesisBlocks, + incompleteBlockRepairBatchSize, + mempoolResync, + mempoolResyncMinIntervalMs, + mempoolTransactionExpirationMs, + mempoolTransactionExpirationScanIntervalMs, postgresMaxConnections, postgresConnectionString, postgresSynchronousCommit, diff --git a/src/db.ts b/src/db.ts index 8c62d188..fb7080b2 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,11 +1,22 @@ +/* eslint-disable max-lines */ import pg from 'pg'; import type { Agent } from './agent.js'; +import { + copyStageTableSql, + createStageTablesSql, + encodeStageBlockTransactions, + encodeStageInputs, + encodeStageOutputs, + encodeStageTransactions, +} from './components/block-copy-rows.js'; import { computeIndexCreationProgress, indexDefinitions, } from './components/db-utils.js'; +import { copyFromBuffers } from './components/pg-binary-copy.js'; import { + chaingraphWritePath, postgresConnectionString, postgresMaxConnections, postgresSynchronousCommit, @@ -83,16 +94,930 @@ export const blockArrayToHashChain = ( */ export const getAllKnownBlockHashes = async () => { const client = await pool.connect(); - const allKnownBlockHashes = await client.query<{ hash: Buffer }>( - 'SELECT "hash" from "block";' + /* + * Hex-encoding in Postgres avoids materializing millions of `Buffer`s and + * converting each to hex on the (single-threaded) agent event loop, which + * dominated startup time on large databases. + */ + const allKnownBlockHashes = await client.query<{ hash: string }>( + `SELECT encode("hash", 'hex') AS "hash" from "block";` ); client.release(); - const hashes = allKnownBlockHashes.rows.map(({ hash }) => - hash.toString('hex') + return allKnownBlockHashes.rows.map(({ hash }) => hash); +}; + +export interface IncompleteBlock { + hash: string; + height: number; + linkedSizeBytes: number; + sizeBytes: number; + transactionCount: number; +} + +export interface IncompleteBlockScan { + incompleteBlocks: IncompleteBlock[]; + scannedBlockCount: number; +} + +export interface ExpiringMempoolTransaction { + expiresAt: Date; + hash: string; + nodeInternalId: number; + nodeName: string; + transactionInternalId: number; + validatedAt: Date; +} + +export interface ArchivedMempoolTransaction { + hash: string; + nodeName: string; + replacedAt: Date | null; +} + +/** + * Find blocks for which the locally saved block_transaction rows don't sum to + * the block's saved byte size. This avoids the SQL block encoder so it can + * detect incomplete blocks even if encoder functions have bugs (e.g. #75). + * + * Only blocks accepted by one of `nodeInternalIds` are checked, and for each + * node only blocks with an `internal_id` above the matching entry of + * `auditedThroughBlockInternalIds` (0 if the node has never completed an + * audit). Summing `transaction.size_bytes` reads every linked transaction row, + * so on a large, already-audited database this filter is what keeps the scan + * from re-reading the whole `transaction` table on every startup. + */ +export const getIncompleteBlocks = async ({ + auditedThroughBlockInternalIds, + heightLowerBound, + heightUpperBound, + limit, + nodeInternalIds, + excludedBlockHashes, +}: { + /** + * Aligned with `nodeInternalIds`; defaults to 0 (audit every block) for + * each node. + */ + auditedThroughBlockInternalIds?: number[]; + excludedBlockHashes: string[]; + heightLowerBound: number; + heightUpperBound: number; + limit: number; + nodeInternalIds: number[]; +}): Promise => { + if (nodeInternalIds.length === 0) { + return { incompleteBlocks: [], scannedBlockCount: 0 }; + } + // cspell:ignore unnest + const auditedThrough = nodeInternalIds.map( + (_, index) => auditedThroughBlockInternalIds?.[index] ?? 0 ); - return hashes; + /* + * If any node has never completed an audit, every block it accepted must be + * audited anyway, so use the original (unbounded) filter: on a full audit + * it is measurably cheaper than the per-node join (measured on an 831k-block + * mainnet checkpoint: 413 s vs 552 s for the same bytes read). The per-node bounded filter is used + * only when every node has a marker. + */ + const minimumAuditedThrough = Math.min(...auditedThrough); + const auditEveryBlock = minimumAuditedThrough === 0; + const acceptedByNodeFilter = auditEveryBlock + ? /* sql */ ` + AND EXISTS ( + SELECT 1 FROM node_block + WHERE node_block.block_internal_id = block.internal_id + AND node_block.node_internal_id = ANY($1::integer[]) + )` + : /* sql */ ` + AND block.internal_id > $7 + AND EXISTS ( + SELECT 1 FROM node_block + INNER JOIN unnest($1::integer[], $6::bigint[]) + AS audited_node (node_internal_id, audited_through_block_internal_id) + ON audited_node.node_internal_id = node_block.node_internal_id + WHERE node_block.block_internal_id = block.internal_id + AND block.internal_id > + audited_node.audited_through_block_internal_id + )`; + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const incompleteBlockScan = await client.query<{ + incompleteBlocks: { + hash: string; + height: number | string; + linkedSizeBytes: number | string; + sizeBytes: number | string; + transactionCount: number | string; + }[]; + scannedBlockCount: string; + }>( + /* sql */ ` +WITH linked_transactions AS ( + SELECT + block.internal_id, + block.height, + block.hash, + block.size_bytes, + COUNT(block_transaction.transaction_internal_id)::bigint + AS transaction_count, + COALESCE(SUM(transaction.size_bytes), 0)::bigint + AS transaction_size_bytes + FROM block + LEFT JOIN block_transaction + ON block_transaction.block_internal_id = block.internal_id + LEFT JOIN transaction + ON transaction.internal_id = block_transaction.transaction_internal_id + WHERE block.height >= $2 + AND block.height < $3 + AND NOT (encode(block.hash, 'hex') = ANY($5::text[]))${acceptedByNodeFilter} + GROUP BY block.internal_id +), +linked_block_sizes AS ( + SELECT + hash, + height, + size_bytes, + transaction_count, + 80 + + CASE + WHEN transaction_count <= 252 THEN 1 + WHEN transaction_count <= 65535 THEN 3 + WHEN transaction_count <= 4294967295 THEN 5 + ELSE 9 + END + + transaction_size_bytes AS linked_size_bytes + FROM linked_transactions +), +incomplete_blocks AS ( + SELECT + encode(hash, 'hex') AS hash, + height, + linked_size_bytes, + size_bytes, + transaction_count + FROM linked_block_sizes + WHERE linked_size_bytes != size_bytes + ORDER BY height ASC, hash ASC + LIMIT $4 +) +SELECT + (SELECT COUNT(*)::bigint FROM linked_block_sizes) AS "scannedBlockCount", + COALESCE( + ( + SELECT jsonb_agg( + jsonb_build_object( + 'hash', hash, + 'height', height, + 'linkedSizeBytes', linked_size_bytes, + 'sizeBytes', size_bytes, + 'transactionCount', transaction_count + ) + ORDER BY height ASC, hash ASC + ) + FROM incomplete_blocks + ), + '[]'::jsonb + ) AS "incompleteBlocks"; +`, + [ + nodeInternalIds, + heightLowerBound, + heightUpperBound, + limit, + excludedBlockHashes, + ...(auditEveryBlock ? [] : [auditedThrough, minimumAuditedThrough]), + ] + ); + const scan = incompleteBlockScan.rows[0]!; + return { + incompleteBlocks: scan.incompleteBlocks.map((block) => ({ + hash: block.hash, + height: Number(block.height), + linkedSizeBytes: Number(block.linkedSizeBytes), + sizeBytes: Number(block.sizeBytes), + transactionCount: Number(block.transactionCount), + })), + scannedBlockCount: Number(scan.scannedBlockCount), + }; + } finally { + client.release(); + } +}; + +export interface IncompleteBlockAuditState { + /** + * False if the database schema predates the `incomplete_block_audit` table; + * the agent then audits every block on each startup (and saves nothing). + */ + auditTableExists: boolean; + /** + * Per node internal_id, the block internal_id through which a previous audit + * completed. + */ + auditedThroughBlockInternalIds: Map; + /** + * The highest block internal_id currently saved (0 if there are no blocks). + * Recorded as the new audit marker once this audit completes. + */ + maxBlockInternalId: number; +} + +/** + * Load the incomplete block audit markers for the provided nodes, along with + * the current highest block internal_id. + */ +export const getIncompleteBlockAuditState = async ( + nodeInternalIds: number[] +): Promise => { + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const { auditTableExists, maxBlockInternalId } = ( + await client.query<{ + auditTableExists: boolean; + maxBlockInternalId: string; + }>(/* sql */ ` +SELECT + to_regclass('incomplete_block_audit') IS NOT NULL AS "auditTableExists", + COALESCE((SELECT MAX(internal_id) FROM block), 0)::bigint AS "maxBlockInternalId"; +`) + ).rows[0]!; + if (!auditTableExists) { + return { + auditTableExists, + auditedThroughBlockInternalIds: new Map(), + maxBlockInternalId: Number(maxBlockInternalId), + }; + } + const markers = await client.query<{ + auditedThroughBlockInternalId: string; + nodeInternalId: number; + }>( + /* sql */ ` +SELECT node_internal_id AS "nodeInternalId", + audited_through_block_internal_id AS "auditedThroughBlockInternalId" + FROM incomplete_block_audit + WHERE node_internal_id = ANY($1::integer[]); +`, + [nodeInternalIds] + ); + return { + auditTableExists, + auditedThroughBlockInternalIds: new Map( + markers.rows.map((row) => [ + row.nodeInternalId, + Number(row.auditedThroughBlockInternalId), + ]) + ), + maxBlockInternalId: Number(maxBlockInternalId), + }; + } finally { + client.release(); + } }; +/** + * Record that every block accepted by each of `nodeInternalIds` with an + * internal_id at or below `auditedThroughBlockInternalId` has been audited. + * Markers never move backwards. + */ +export const saveIncompleteBlockAudit = async ({ + auditedThroughBlockInternalId, + nodeInternalIds, +}: { + auditedThroughBlockInternalId: number; + nodeInternalIds: number[]; +}) => { + if (nodeInternalIds.length === 0) { + return; + } + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query( + /* sql */ ` +INSERT INTO incomplete_block_audit + (node_internal_id, audited_through_block_internal_id, completed_at) + SELECT node_internal_id, $2::bigint, now() + FROM unnest($1::integer[]) AS audited_node (node_internal_id) + ON CONFLICT (node_internal_id) DO UPDATE + SET audited_through_block_internal_id = GREATEST( + incomplete_block_audit.audited_through_block_internal_id, + EXCLUDED.audited_through_block_internal_id + ), + completed_at = EXCLUDED.completed_at; +`, + [nodeInternalIds, auditedThroughBlockInternalId] + ); + } finally { + client.release(); + } +}; + +/** + * Find node_transaction rows which will expire before the provided timestamp. + */ +export const getMempoolTransactionsExpiringBefore = async ({ + expiresBefore, + expirationMs, +}: { + expirationMs: number; + expiresBefore: Date; +}): Promise => { + const expirationInterval = `${expirationMs}::double precision * interval '1 millisecond'`; + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const transactions = await client.query<{ + expiresAt: string; + hash: string; + nodeInternalId: string; + nodeName: string; + transactionInternalId: string; + validatedAt: string; + }>(/* sql */ ` +SELECT encode(transaction.hash, 'hex') AS "hash", + node.name AS "nodeName", + node_transaction.node_internal_id AS "nodeInternalId", + node_transaction.transaction_internal_id AS "transactionInternalId", + node_transaction.validated_at::text AS "validatedAt", + (node_transaction.validated_at + (${expirationInterval}))::text AS "expiresAt" + FROM node_transaction + JOIN node + ON node.internal_id = node_transaction.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + WHERE node_transaction.validated_at + (${expirationInterval}) <= ${dateToTimestampWithoutTimezone( + expiresBefore + )} + ORDER BY "expiresAt", "nodeName", "hash"; +`); + return transactions.rows.map((transaction) => ({ + expiresAt: timestampWithoutTimezoneToDate(transaction.expiresAt), + hash: transaction.hash, + nodeInternalId: Number(transaction.nodeInternalId), + nodeName: transaction.nodeName, + transactionInternalId: Number(transaction.transactionInternalId), + validatedAt: timestampWithoutTimezoneToDate(transaction.validatedAt), + })); + } finally { + client.release(); + } +}; + +/** + * Archive node_transaction rows for transactions that are already accepted or + * replaced by accepted blocks for the same node. This repairs historical rows + * missed when block inclusions are added after the node_block trigger has + * already fired. + * + * The correlated input lookups use OFFSET 0 as a planning barrier. Without it, + * Postgres can flatten the joins and scan the entire historical input table to + * resolve conflicts for a comparatively small current mempool. Keep each lookup + * constrained to a mempool transaction and then one of its outpoints. + */ +export const archiveMempoolTransactionsAcceptedByBlocks = async (): Promise< + ArchivedMempoolTransaction[] +> => { + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + const result = await client.query<{ + hash: string; + nodeName: string; + replacedAt: string | null; + }>(/* sql */ ` +WITH directly_accepted AS ( + SELECT node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + NULL::timestamp without time zone AS replaced_at + FROM node_transaction + JOIN block_transaction + ON block_transaction.transaction_internal_id = node_transaction.transaction_internal_id + JOIN node_block + ON node_block.node_internal_id = node_transaction.node_internal_id + AND node_block.block_internal_id = block_transaction.block_internal_id +), +replaced_by_accepted AS ( + SELECT node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + replacement.replaced_at + FROM node_transaction + CROSS JOIN LATERAL ( + SELECT MIN(node_block.accepted_at) AS replaced_at + FROM LATERAL ( + SELECT outpoint_transaction_hash, outpoint_index + FROM input + WHERE transaction_internal_id = node_transaction.transaction_internal_id + AND outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea + OFFSET 0 + ) mempool_input + CROSS JOIN LATERAL ( + SELECT transaction_internal_id + FROM input + WHERE outpoint_transaction_hash = mempool_input.outpoint_transaction_hash + AND outpoint_index = mempool_input.outpoint_index + AND transaction_internal_id != node_transaction.transaction_internal_id + OFFSET 0 + ) accepted_input + JOIN block_transaction + ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id + JOIN node_block + ON node_block.node_internal_id = node_transaction.node_internal_id + AND node_block.block_internal_id = block_transaction.block_internal_id + HAVING COUNT(*) > 0 + ) replacement +), +archive_candidates AS ( + SELECT node_internal_id, transaction_internal_id, replaced_at + FROM directly_accepted + UNION ALL + SELECT node_internal_id, transaction_internal_id, replaced_at + FROM replaced_by_accepted +), +archive_rows AS ( + SELECT node_internal_id, + transaction_internal_id, + CASE + WHEN bool_or(replaced_at IS NULL) THEN NULL::timestamp without time zone + ELSE MIN(replaced_at) + END AS replaced_at + FROM archive_candidates + GROUP BY node_internal_id, transaction_internal_id +), +deleted_rows AS ( + DELETE FROM node_transaction + USING archive_rows + WHERE node_transaction.node_internal_id = archive_rows.node_internal_id + AND node_transaction.transaction_internal_id = archive_rows.transaction_internal_id + RETURNING node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + node_transaction.validated_at, + archive_rows.replaced_at +), +inserted_history AS ( + INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at + FROM deleted_rows + RETURNING node_internal_id, transaction_internal_id, replaced_at +) +SELECT encode(transaction.hash, 'hex') AS "hash", + node.name AS "nodeName", + inserted_history.replaced_at::text AS "replacedAt" + FROM inserted_history + JOIN node + ON node.internal_id = inserted_history.node_internal_id + JOIN transaction + ON transaction.internal_id = inserted_history.transaction_internal_id + ORDER BY "nodeName", "hash"; +`); + return result.rows.map((row) => ({ + hash: row.hash, + nodeName: row.nodeName, + replacedAt: + row.replacedAt === null + ? null + : timestampWithoutTimezoneToDate(row.replacedAt), + })); + } finally { + client.release(); + } +}; + +export interface MempoolTransactionExpiry { + nodeInternalId: number; + replacedAt: Date; + transactionInternalId: number; + /** + * The `validated_at` of the validation this expiry was scheduled for. When + * set, a row validated again later (e.g. restored by a mempool resync after a + * re-org) is left alone; it expires at its own deadline. + */ + validatedAt?: Date; +} + +// cspell:words xact unnest +const deadlockDetectedCode = '40P01'; +const serializationFailureCode = '40001'; +/** A descendant walked by the archival statement was deleted before its lock. */ +const descendantVanishedCode = 'chaingraph-expiry-descendant-vanished'; +const isRetriedConflict = (err: unknown) => + [ + deadlockDetectedCode, + serializationFailureCode, + descendantVanishedCode, + ].includes((err as { code?: string } | undefined)?.code ?? ''); +const expiryRetryLimit = 6; +const expiryRetryBaseMs = 50; +const expiryBatchSize = 500; +/** + * Advisory lock key serializing mempool expiry across every agent sharing the + * database (the agent also runs expiry batches one at a time). + */ +export const expiryAdvisoryLockKey = 1970172700; + +/** + * Resolution of mempool transaction `transaction` for node `node` by the blocks + * that node accepted, as recorded by + * {@link archiveMempoolTransactionsAcceptedByBlocks}: `accepted` if the + * transaction is in one of them; `replaced` if one of them spends one of its + * outpoints, with `replaced_at` the earliest such block's accepted_at (NULL if + * unknown: blocks accepted before Chaingraph began monitoring). + */ +const acceptedBlockResolution = (node: string, transaction: string) => { + const conflictingAcceptedSpends = /* sql */ ` + FROM input mempool_input + JOIN input accepted_input + ON accepted_input.outpoint_transaction_hash = mempool_input.outpoint_transaction_hash + AND accepted_input.outpoint_index = mempool_input.outpoint_index + AND accepted_input.transaction_internal_id != ${transaction} + JOIN block_transaction + ON block_transaction.transaction_internal_id = accepted_input.transaction_internal_id + JOIN node_block + ON node_block.node_internal_id = ${node} + AND node_block.block_internal_id = block_transaction.block_internal_id + WHERE mempool_input.transaction_internal_id = ${transaction} + AND mempool_input.outpoint_transaction_hash != '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea`; + return /* sql */ ` + SELECT EXISTS ( + SELECT 1 + FROM block_transaction + JOIN node_block + ON node_block.node_internal_id = ${node} + AND node_block.block_internal_id = block_transaction.block_internal_id + WHERE block_transaction.transaction_internal_id = ${transaction} + ) AS accepted, + EXISTS (SELECT 1 ${conflictingAcceptedSpends} + ) AS replaced, + (SELECT MIN(node_block.accepted_at) ${conflictingAcceptedSpends} + ) AS replaced_at`; +}; + +/** + * The statement archiving a batch of expired node_transaction rows and their + * same-node descendants (see {@link archiveMempoolTransactions}): `$1` to `$4` + * are the batch's node internal IDs, transaction internal IDs, replaced_at + * timestamps and scheduled validated_at timestamps (NULL to skip that check). + * + * Each step of the descendant walk looks up the spending transaction's + * node_transaction row by its full key behind an OFFSET 0 planning barrier. + * Without it, Postgres can join the walked rows to node_transaction by node + * alone (then filter by the spending inputs), reading the node's whole mempool + * once per walked row: it does so when node_transaction statistics predate + * the current mempool (e.g. last analyzed just after a block emptied it), so + * archiving a large mempool would take time quadratic in its size. + */ +export const mempoolExpiryArchivalStatement = /* sql */ ` +WITH RECURSIVE expiry AS ( + SELECT node_internal_id, transaction_internal_id, replaced_at, validated_at + FROM unnest($1::integer[], $2::bigint[], $3::timestamp[], $4::timestamp[]) + AS expiry (node_internal_id, transaction_internal_id, replaced_at, validated_at) +), +expired AS ( + SELECT expiry.node_internal_id, + expiry.transaction_internal_id, + expiry.replaced_at + FROM expiry + JOIN node_transaction + ON node_transaction.node_internal_id = expiry.node_internal_id + AND node_transaction.transaction_internal_id = expiry.transaction_internal_id + CROSS JOIN LATERAL (${acceptedBlockResolution( + 'expiry.node_internal_id', + 'expiry.transaction_internal_id' + )} + ) resolution + WHERE NOT resolution.accepted + AND NOT resolution.replaced + -- validated_at has microseconds; the scheduled Date has milliseconds + AND (expiry.validated_at IS NULL + OR date_trunc('milliseconds', node_transaction.validated_at) <= expiry.validated_at) +), +invalidated (node_internal_id, transaction_internal_id, replaced_at) AS ( + SELECT node_internal_id, transaction_internal_id, replaced_at + FROM expired + WHERE NOT EXISTS ( + SELECT 1 + FROM input + JOIN transaction parent_transaction + ON parent_transaction.hash = input.outpoint_transaction_hash + JOIN expired expired_parent + ON expired_parent.node_internal_id = expired.node_internal_id + AND expired_parent.transaction_internal_id = parent_transaction.internal_id + AND expired_parent.replaced_at <= expired.replaced_at + WHERE input.transaction_internal_id = expired.transaction_internal_id + ) + UNION + SELECT child.node_internal_id, + child.transaction_internal_id, + CASE + WHEN resolution.accepted THEN NULL::timestamp without time zone + WHEN resolution.replaced THEN resolution.replaced_at + ELSE parent.replaced_at + END + FROM invalidated parent + JOIN transaction parent_transaction + ON parent_transaction.internal_id = parent.transaction_internal_id + JOIN input child_input + ON child_input.outpoint_transaction_hash = parent_transaction.hash + CROSS JOIN LATERAL ( + SELECT node_internal_id, transaction_internal_id + FROM node_transaction + WHERE node_internal_id = parent.node_internal_id + AND transaction_internal_id = child_input.transaction_internal_id + OFFSET 0 + ) child + CROSS JOIN LATERAL (${acceptedBlockResolution( + 'child.node_internal_id', + 'child.transaction_internal_id' + )} + ) resolution + WHERE parent.replaced_at IS NOT NULL +), +archive_rows AS ( + -- confirmation takes precedence, then the earliest invalidation + SELECT node_internal_id, + transaction_internal_id, + CASE + WHEN bool_or(replaced_at IS NULL) THEN NULL::timestamp without time zone + ELSE MIN(replaced_at) + END AS replaced_at + FROM invalidated + GROUP BY node_internal_id, transaction_internal_id +), +locked AS ( + SELECT node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + archive_rows.replaced_at + FROM archive_rows + JOIN node_transaction + ON node_transaction.node_internal_id = archive_rows.node_internal_id + AND node_transaction.transaction_internal_id = archive_rows.transaction_internal_id + ORDER BY node_transaction.node_internal_id, node_transaction.transaction_internal_id + FOR UPDATE OF node_transaction +), +-- false if a walked row was deleted before its lock (the walk may be stale) +complete AS ( + SELECT (SELECT count(*) FROM locked) = (SELECT count(*) FROM archive_rows) AS complete +), +deleted_rows AS ( + DELETE FROM node_transaction + USING locked + WHERE (SELECT complete FROM complete) + AND node_transaction.node_internal_id = locked.node_internal_id + AND node_transaction.transaction_internal_id = locked.transaction_internal_id + RETURNING node_transaction.node_internal_id, + node_transaction.transaction_internal_id, + node_transaction.validated_at, + locked.replaced_at +), +inserted_history AS ( + INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT node_internal_id, transaction_internal_id, validated_at, replaced_at + FROM deleted_rows + RETURNING node_internal_id, transaction_internal_id +) +SELECT complete.complete, + inserted_history.node_internal_id || ':' || inserted_history.transaction_internal_id AS archived + FROM complete + LEFT JOIN inserted_history ON true; +`; + +/** + * Archive expired node_transaction rows, and their same-node descendants, in + * one transaction: take the expiry advisory lock, lock the expired rows, then + * archive them in one statement. Each is a separate command, so (in READ + * COMMITTED) the archival statement's snapshot is taken after both waits: it + * sees block inclusions committed while expiry waited, and no expired row + * deleted meanwhile (e.g. confirmed by block acceptance) can seed the + * descendant walk, as rows it has locked cannot be deleted under it. + * Descendants are only locked by the archival statement itself, after its walk: + * if one was deleted meanwhile (e.g. confirmed, ending the cascade there), the + * walk may have reached its own descendants through it, so the statement + * deletes nothing and the transaction is retried with a fresh snapshot. + * + * An expired row is left in place (not returned) if the transaction, or a + * conflicting spend of one of its outpoints, is already in a block accepted by + * the same node: {@link archiveMempoolTransactionsAcceptedByBlocks} archives + * those with the correct replaced_at (NULL or the block's accepted_at). The + * guard is part of the statement that locks and deletes the row, so it is + * evaluated against the row being archived, not a read taken earlier: a row + * inserted after the agent's accepted-block archive but before its expiry scan + * would otherwise be archived as expired/replaced even though it is confirmed. + * (Such a row is still archived if it descends from another row archived + * here, with the replaced_at described below.) + * + * Descendants are archived here rather than by the history trigger's cascade + * (which then finds none left), for two reasons: + * - a descendant gets the earliest expiry among its archived ancestors, also + * when it is itself in the batch with a later expiry (the trigger only + * reaches descendants still in node_transaction, so it cannot carry an + * ancestor's expiry through a descendant this statement deletes itself); + * - a descendant resolved by a block accepted by the node gets the + * replaced_at the accepted-block archive would give it (NULL if confirmed, + * ending the cascade there; otherwise the block's accepted_at, passed on to + * its own descendants, or NULL if that is unknown, again ending the cascade) + * instead of its ancestor's expiry. + * + * Only expired rows without an expired parent expiring no later seed the + * cascade (the others are reached from that parent with an earlier or equal + * replaced_at), so an expired chain is walked once rather than once per row. + * + * Deadlocks: archiving a mempool transaction also archives its descendants, so + * concurrent single-row expiries of a parent and of its descendants lock the + * same rows in opposite orders (each locks its own row, then the cascade waits + * for the other's) and deadlock; a restarted agent's stale mempool, expiring + * all at once on every pool connection, live-locked this way. Expiry therefore + * runs one batch at a time (see {@link expireMempoolTransaction}), under a + * transaction-level advisory lock (also across agents sharing the database), + * locks the rows it archives in (node, transaction) order (the expired rows, + * then their descendants) before deleting them, and retries a deadlock, + * serialization failure or vanished descendant (see above) against other + * writers (block acceptance, double-spend replacement) with backoff; once + * retries are exhausted it throws, leaving the rows for the next expiration + * scan. + */ +export const archiveMempoolTransactions = async ( + expiries: MempoolTransactionExpiry[] +): Promise => { + if (expiries.length === 0) { + return []; + } + const sorted = [...expiries].sort( + (a, b) => + a.nodeInternalId - b.nodeInternalId || + a.transactionInternalId - b.transactionInternalId + ); + const nodeInternalIds = sorted.map((expiry) => expiry.nodeInternalId); + const transactionInternalIds = sorted.map( + (expiry) => expiry.transactionInternalId + ); + const archiveOnce = async () => { + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query('BEGIN;'); + await client.query('SELECT pg_advisory_xact_lock($1::bigint);', [ + expiryAdvisoryLockKey, + ]); + await client.query( + /* sql */ ` +SELECT 1 + FROM node_transaction + JOIN unnest($1::integer[], $2::bigint[]) + AS expiry (node_internal_id, transaction_internal_id) + ON node_transaction.node_internal_id = expiry.node_internal_id + AND node_transaction.transaction_internal_id = expiry.transaction_internal_id + ORDER BY node_transaction.node_internal_id, node_transaction.transaction_internal_id + FOR UPDATE OF node_transaction; +`, + [nodeInternalIds, transactionInternalIds] + ); + const result = await client.query<{ + archived: string | null; + complete: boolean; + }>(mempoolExpiryArchivalStatement, [ + nodeInternalIds, + transactionInternalIds, + sorted.map((expiry) => expiry.replacedAt.toISOString()), + sorted.map((expiry) => expiry.validatedAt?.toISOString() ?? null), + ]); + if (result.rows[0]?.complete !== true) { + // eslint-disable-next-line functional/no-throw-statement + throw Object.assign( + new Error( + 'Mempool expiry: a descendant was deleted while the expiry waited for its lock.' + ), + { code: descendantVanishedCode } + ); + } + await client.query('COMMIT;'); + const archived = new Set(result.rows.map((row) => row.archived)); + return sorted.filter((expiry) => + archived.has(`${expiry.nodeInternalId}:${expiry.transactionInternalId}`) + ); + } catch (err) { + await client.query('ROLLBACK;'); + // eslint-disable-next-line functional/no-throw-statement + throw err; + } finally { + client.release(); + } + }; + const attempt = async (retry: number): Promise => + archiveOnce().catch(async (err: unknown) => { + if (!isRetriedConflict(err) || retry >= expiryRetryLimit) { + // eslint-disable-next-line functional/no-throw-statement + throw err; + } + // eslint-disable-next-line @typescript-eslint/no-magic-numbers + const backoffMs = expiryRetryBaseMs * 2 ** retry * (1 + Math.random()); + await new Promise((resolve) => { + setTimeout(resolve, backoffMs); + }); + return attempt(retry + 1); + }); + return attempt(0); +}; + +/** + * Archive a single expired node_transaction row (see + * {@link archiveMempoolTransactions}); returns the number archived (0 or 1). + */ +export const archiveMempoolTransaction = async ( + expiry: MempoolTransactionExpiry +) => (await archiveMempoolTransactions([expiry])).length; + +const pendingExpiries = new Map< + string, + { + expiry: MempoolTransactionExpiry; + settle: { + reject: (err: unknown) => void; + resolve: (archived: boolean) => void; + }[]; + } +>(); +// eslint-disable-next-line functional/no-let, @typescript-eslint/init-declarations +let expiryDrain: Promise | undefined; + +const drainExpiries = async (): Promise => { + /* + * Earliest first: an ancestor's expiry is never in a later batch than its + * descendant's (which would then be archived with its own, later expiry). + */ + const batch = [...pendingExpiries.entries()] + .sort( + ([, a], [, b]) => + a.expiry.replacedAt.getTime() - b.expiry.replacedAt.getTime() + ) + .slice(0, expiryBatchSize); + if (batch.length === 0) { + return undefined; + } + batch.forEach(([key]) => pendingExpiries.delete(key)); + await archiveMempoolTransactions(batch.map(([, entry]) => entry.expiry)) + .then((archived) => { + const archivedKeys = new Set( + archived.map( + (expiry) => `${expiry.nodeInternalId}:${expiry.transactionInternalId}` + ) + ); + batch.forEach(([key, entry]) => { + entry.settle.forEach(({ resolve }) => { + resolve(archivedKeys.has(key)); + }); + }); + }) + .catch((err: unknown) => { + batch.forEach(([, entry]) => { + entry.settle.forEach(({ reject }) => { + reject(err); + }); + }); + }); + return drainExpiries(); +}; + +const startExpiryDrain = () => { + if (expiryDrain !== undefined) { + return; + } + expiryDrain = new Promise((started) => { + // let every timer firing in this tick join the first batch + setImmediate(started); + }) + .then(drainExpiries) + .finally(() => { + expiryDrain = undefined; + if (pendingExpiries.size > 0) { + startExpiryDrain(); + } + }); +}; + +/** + * Queue an expired mempool transaction for archiving; resolves `true` if it was + * archived, `false` if it was left in place (already gone, or accepted by a + * block for the node). Expiries run one batch at a time (up to + * {@link expiryBatchSize} rows per statement): when many fire at once (a + * restarted agent's stale mempool) they are archived in a few ordered + * statements on one connection instead of one racing statement per pool + * connection. + */ +export const expireMempoolTransaction = async ( + expiry: MempoolTransactionExpiry +) => + new Promise((resolve, reject) => { + const key = `${expiry.nodeInternalId}:${expiry.transactionInternalId}`; + const pending = pendingExpiries.get(key); + if (pending === undefined) { + pendingExpiries.set(key, { expiry, settle: [{ reject, resolve }] }); + } else { + pending.settle.push({ reject, resolve }); + } + startExpiryDrain(); + }); /** * Create or update one or more trusted node in the Chaingraph database, * returning it's internal ID. @@ -200,15 +1125,6 @@ WITH transaction_values (hash, version, locktime, size_bytes, is_coinbase) AS ( )}'::bytea, '${hexToByteaString(input.unlockingBytecode)}'::bytea)` ) .join(',')} -), node_transaction_values (node_internal_id, validated_at) AS ( - VALUES ${nodeValidations - .map( - (validation) => - `(${ - validation.nodeInternalId - }::bigint, ${dateToTimestampWithoutTimezone(validation.validatedAt)})` - ) - .join(',')} ), new_transaction (transaction_hash, transaction_internal_id) AS ( INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) SELECT hash, version, locktime, size_bytes, is_coinbase FROM transaction_values @@ -220,27 +1136,57 @@ WITH transaction_values (hash, version, locktime, size_bytes, is_coinbase) AS ( ), insert_inputs AS ( INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) SELECT transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode FROM input_values CROSS JOIN new_transaction -), new_or_existing_transaction (transaction_internal_id) AS ( - SELECT COALESCE ( - (SELECT transaction_internal_id FROM new_transaction), - (SELECT internal_id AS transaction_internal_id FROM transaction WHERE transaction.hash = '${hexToByteaString( - transaction.hash - )}'::bytea) - ) +) +SELECT COUNT(*) FROM new_transaction; +`; + const saveNodeValidations = /* sql */ ` +WITH node_transaction_values (node_internal_id, validated_at) AS ( + VALUES ${nodeValidations + .map( + (validation) => + `(${ + validation.nodeInternalId + }::bigint, ${dateToTimestampWithoutTimezone(validation.validatedAt)})` + ) + .join(',')} ) INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) - SELECT node_internal_id, transaction_internal_id, validated_at FROM node_transaction_values CROSS JOIN new_or_existing_transaction; + SELECT node_internal_id, $1::bigint, validated_at FROM node_transaction_values + ON CONFLICT ON CONSTRAINT "node_transaction_pkey" DO NOTHING; `; const client = await pool.connect(); - await client.query(saveTransaction); - client.release(); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query('BEGIN;'); + await client.query(saveTransaction); + const transactionInternalIdResult = await client.query<{ + internalId: string; + }>( + /* sql */ `SELECT internal_id AS "internalId" FROM transaction WHERE hash = $1;`, + [Buffer.from(transaction.hash, 'hex')] + ); + const transactionInternalId = + transactionInternalIdResult.rows[0]?.internalId; + if (transactionInternalId === undefined) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Failed to save or find transaction while recording node validation: ${transaction.hash}` + ); + } + await client.query(saveNodeValidations, [transactionInternalId]); + await client.query('COMMIT;'); + } catch (err) { + await client.query('ROLLBACK;'); + // eslint-disable-next-line functional/no-throw-statement + throw err; + } finally { + client.release(); + } }; /** * Immediately mark a node as having validated a transaction already known to * exist in the database. - * - * TODO: test */ export const recordNodeValidation = async ( transactionHash: string, @@ -253,16 +1199,246 @@ export const recordNodeValidation = async ( /* * The transaction is already saved, just insert `node_transaction`s. */ - await client.query(/* sql */ ` - INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) - SELECT node_internal_id, validated_at FROM (VALUES ( + // eslint-disable-next-line functional/no-try-statement + try { + await client.query(/* sql */ ` + WITH node_transaction_values (node_internal_id, validated_at) AS ( + VALUES ( ${validation.nodeInternalId}::bigint, - ${dateToTimestampWithoutTimezone(validation.validatedAt)}) - INNER JOIN (SELECT internal_id as transaction_internal_id from transaction WHERE hash = '${hexToByteaString( - transactionHash - )}'::bytea); - `); - client.release(); + ${dateToTimestampWithoutTimezone(validation.validatedAt)} + ) + ), known_transaction (transaction_internal_id) AS ( + SELECT internal_id + FROM transaction + WHERE hash = '${hexToByteaString(transactionHash)}'::bytea + ) + INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT node_internal_id, transaction_internal_id, validated_at + FROM node_transaction_values + CROSS JOIN known_transaction + ON CONFLICT ON CONSTRAINT "node_transaction_pkey" DO NOTHING; + `); + } finally { + client.release(); + } +}; + +/** + * Inside a block's DB transaction: throw (so the caller rolls back) unless + * every transaction of the block was found and linked via block_transaction. + */ +const verifyBlockTransactionsLinked = async ( + client: pg.PoolClient, + block: ChaingraphBlock, + joinedTransactionCount: number +) => { + const linkedBlockTransactionCount = Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*)::bigint AS count + FROM block_transaction + INNER JOIN block ON block.internal_id = block_transaction.block_internal_id + WHERE block.hash = $1; + `, + [Buffer.from(block.hash, 'hex')] + ) + ).rows[0]!.count + ); + if ( + joinedTransactionCount !== block.transactions.length || + linkedBlockTransactionCount !== block.transactions.length + ) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Failed to save all transactions for block ${block.height} (${block.hash}): joined ${joinedTransactionCount}/${block.transactions.length}, linked ${linkedBlockTransactionCount}/${block.transactions.length}.` + ); + } +}; + +/** + * Pooled connections on which the `copy` write path's staging tables exist. + * (A replaced connection is a new client object, so it is set up again.) + */ +const clientsWithStageTables = new WeakSet(); + +/** + * The `copy` write path's statement for transactions, outputs and inputs: the + * same CTE as `addAllTransactions` in `saveBlock`, reading the staged rows + * instead of `VALUES` lists. The text never changes, so Postgres parses it + * cheaply; it is deliberately not a named prepared statement, so each block + * is planned with the current size of the staging tables. + */ +const addAllStagedTransactions = /* sql */ ` +WITH newly_saved_transactions (hash, internal_id) AS ( + INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, version, locktime, size_bytes, is_coinbase FROM pg_temp.chaingraph_stage_transaction + ON CONFLICT ON CONSTRAINT "transaction_hash_key" DO NOTHING + RETURNING hash, internal_id +), +newly_saved_outputs AS ( + INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode, token_category, fungible_token_amount, nonfungible_token_capability, nonfungible_token_commitment) + SELECT transaction_hash, output_index, value_satoshis, locking_bytecode, token_category, fungible_token_amount, nonfungible_token_capability, nonfungible_token_commitment FROM pg_temp.chaingraph_stage_output + WHERE transaction_hash IN (SELECT hash FROM newly_saved_transactions) +), +newly_saved_inputs AS ( + INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode + FROM pg_temp.chaingraph_stage_input val INNER JOIN newly_saved_transactions txs ON val.transaction_hash = txs.hash +) +SELECT COUNT(*) FROM newly_saved_transactions;`; + +/** + * The `copy` write path's block statement: the same CTE as `addBlockQuery` in + * `saveBlock`, with the block's transaction list read from the staging table + * and every other value passed as a parameter (so the text is fixed). + * + * `joined_transactions` is an index lookup per staged transaction (the + * `OFFSET 0` keeps the lateral subquery from being flattened into a join). + * Temporary tables have no statistics, and Postgres assumes at least 10 pages + * for a never-vacuumed table, so a small block's list is estimated at ~1,200 + * rows; with a free choice the planner can pick a hash join over a full scan + * of `transaction`. + */ +const addStagedBlock = /* sql */ ` +WITH accepting_nodes (node_internal_id, accepted_at) AS ( + SELECT * FROM unnest($10::bigint[], $11::timestamp[]) +), +joined_transactions (internal_id, transaction_index) AS ( + SELECT db.internal_id, val.transaction_index + FROM pg_temp.chaingraph_stage_block_transaction val + CROSS JOIN LATERAL ( + SELECT internal_id FROM transaction WHERE transaction.hash = val.hash OFFSET 0 + ) db +), +inserted_block (internal_id) AS ( + INSERT INTO block (height, version, timestamp, hash, previous_block_hash, merkle_root, bits, nonce, size_bytes) + VALUES ($1::bigint, $2::bigint, $3::bigint, $4::bytea, $5::bytea, $6::bytea, $7::bigint, $8::bigint, $9::bigint) + ON CONFLICT ON CONSTRAINT "block_hash_key" DO NOTHING + RETURNING internal_id +), +new_or_existing_block (internal_id) AS ( + SELECT COALESCE ( + (SELECT internal_id FROM inserted_block), + (SELECT internal_id FROM block WHERE block.hash = $4::bytea) + ) +), +inserted_block_transactions AS ( + INSERT INTO block_transaction (block_internal_id, transaction_internal_id, transaction_index) + SELECT blk.internal_id, tx.internal_id, tx.transaction_index + FROM new_or_existing_block blk CROSS JOIN joined_transactions tx + ON CONFLICT ON CONSTRAINT "block_transaction_pkey" DO NOTHING + RETURNING transaction_internal_id +), +inserted_node_blocks AS ( + INSERT INTO node_block (node_internal_id, block_internal_id, accepted_at) + SELECT node.node_internal_id, blk.internal_id, node.accepted_at + FROM new_or_existing_block blk CROSS JOIN accepting_nodes node + ON CONFLICT ON CONSTRAINT "node_block_pkey" DO NOTHING + RETURNING block_internal_id +) +SELECT + (SELECT COUNT(*)::bigint FROM joined_transactions) AS "joinedTransactionCount", + (SELECT COUNT(*)::bigint FROM inserted_block_transactions) AS "insertedBlockTransactionCount", + (SELECT COUNT(*)::bigint FROM inserted_node_blocks) AS "insertedNodeBlockCount";`; + +/** + * The `copy` write path (`CHAINGRAPH_WRITE_PATH=copy`) of `saveBlock`: one DB + * transaction per block, as in the `sql` path. Rows are streamed with binary + * `COPY` into per-connection temporary staging tables, then inserted with the + * same statements, constraints and `ON CONFLICT DO NOTHING` handling. + */ +const saveBlockViaCopy = async ({ + block, + nodeAcceptances, + unknownTransactions, +}: { + block: ChaingraphBlock; + nodeAcceptances: { + nodeInternalId: number; + acceptedAt: Date | null; + }[]; + unknownTransactions: ChaingraphTransaction[]; +}) => { + const stagedTransactions = encodeStageTransactions(unknownTransactions); + const stagedOutputs = encodeStageOutputs(unknownTransactions); + const stagedInputs = encodeStageInputs(unknownTransactions); + const stagedBlockTransactions = encodeStageBlockTransactions(block); + const blockParameters = [ + block.height, + block.version, + block.timestamp, + Buffer.from(block.hash, 'hex'), + Buffer.from(block.previousBlockHash, 'hex'), + Buffer.from(block.merkleRoot, 'hex'), + block.bits, + block.nonce, + block.sizeBytes, + nodeAcceptances.map((acceptance) => acceptance.nodeInternalId), + nodeAcceptances.map((acceptance) => + acceptance.acceptedAt === null + ? null + : acceptance.acceptedAt.toISOString() + ), + ]; + const client = await pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + if (!clientsWithStageTables.has(client)) { + /* + * Outside of the block's DB transaction, so a rolled-back block can't + * also roll back the table creation. + */ + await client.query(createStageTablesSql); + clientsWithStageTables.add(client); + } + await client.query('BEGIN;'); + await copyFromBuffers(client, [ + { + messages: stagedTransactions, + statement: copyStageTableSql('chaingraph_stage_transaction'), + }, + { + messages: stagedOutputs, + statement: copyStageTableSql('chaingraph_stage_output'), + }, + { + messages: stagedInputs, + statement: copyStageTableSql('chaingraph_stage_input'), + }, + { + messages: stagedBlockTransactions, + statement: copyStageTableSql('chaingraph_stage_block_transaction'), + }, + ]); + const saveTransactionsResult = await client.query<{ count: string }>( + addAllStagedTransactions + ); + const savedTransactionCount = Number(saveTransactionsResult.rows[0]!.count); + const transactionCacheMisses = + unknownTransactions.length - savedTransactionCount; + const addBlockResult = await client.query<{ + insertedBlockTransactionCount: string; + insertedNodeBlockCount: string; + joinedTransactionCount: string; + }>(addStagedBlock, blockParameters); + await verifyBlockTransactionsLinked( + client, + block, + Number(addBlockResult.rows[0]!.joinedTransactionCount) + ); + await client.query('COMMIT;'); + return { + attemptedSavedTransactions: unknownTransactions, + transactionCacheMisses, + }; + } catch (err) { + await client.query('ROLLBACK;'); + // eslint-disable-next-line functional/no-throw-statement + throw err; + } finally { + client.release(); + } }; /** @@ -300,7 +1476,7 @@ export const saveBlock = async ({ }>( (transactions, transaction) => { // eslint-disable-next-line @typescript-eslint/no-unused-expressions - transactionCache.has(transaction.hash) + transactionCache.get(transaction.hash)?.db === true ? transactions.alreadySaved.push(transaction) : transactions.unknown.push(transaction); return transactions; @@ -308,6 +1484,14 @@ export const saveBlock = async ({ { alreadySaved: [], unknown: [] } ); + if (chaingraphWritePath === 'copy') { + return saveBlockViaCopy({ + block, + nodeAcceptances, + unknownTransactions: blockTransactions.unknown, + }); + } + const inputs: { inputIndex: number; transactionHash: string; @@ -453,11 +1637,6 @@ inserted_block (internal_id) AS ( ON CONFLICT ON CONSTRAINT "block_hash_key" DO NOTHING RETURNING internal_id ), -inserted_block_transactions AS ( - INSERT INTO block_transaction (block_internal_id, transaction_internal_id, transaction_index) - SELECT blk.internal_id, tx.internal_id, tx.transaction_index - FROM inserted_block blk CROSS JOIN joined_transactions tx -), new_or_existing_block (internal_id) AS ( SELECT COALESCE ( (SELECT internal_id FROM inserted_block), @@ -465,27 +1644,58 @@ new_or_existing_block (internal_id) AS ( block.hash )}'::bytea) ) -) -INSERT INTO node_block (node_internal_id, block_internal_id, accepted_at) +), +inserted_block_transactions AS ( + INSERT INTO block_transaction (block_internal_id, transaction_internal_id, transaction_index) + SELECT blk.internal_id, tx.internal_id, tx.transaction_index + FROM new_or_existing_block blk CROSS JOIN joined_transactions tx + ON CONFLICT ON CONSTRAINT "block_transaction_pkey" DO NOTHING + RETURNING transaction_internal_id +), +inserted_node_blocks AS ( + INSERT INTO node_block (node_internal_id, block_internal_id, accepted_at) SELECT node.node_internal_id, blk.internal_id, node.accepted_at FROM new_or_existing_block blk CROSS JOIN accepting_nodes node - ON CONFLICT ON CONSTRAINT "node_block_pkey" DO NOTHING`; + ON CONFLICT ON CONSTRAINT "node_block_pkey" DO NOTHING + RETURNING block_internal_id +) +SELECT + (SELECT COUNT(*)::bigint FROM joined_transactions) AS "joinedTransactionCount", + (SELECT COUNT(*)::bigint FROM inserted_block_transactions) AS "insertedBlockTransactionCount", + (SELECT COUNT(*)::bigint FROM inserted_node_blocks) AS "insertedNodeBlockCount";`; const client = await pool.connect(); - await client.query('BEGIN;'); - const saveTransactionsResult = await client.query<{ count: string }>( - addAllTransactions - ); - const attemptedSavedTransactions = blockTransactions.unknown; - const savedTransactionCount = Number(saveTransactionsResult.rows[0]!.count); - const transactionCacheMisses = - attemptedSavedTransactions.length - savedTransactionCount; - await client.query(addBlockQuery); - await client.query('COMMIT;'); - client.release(); - return { - attemptedSavedTransactions, - transactionCacheMisses, - }; + // eslint-disable-next-line functional/no-try-statement + try { + await client.query('BEGIN;'); + const saveTransactionsResult = await client.query<{ count: string }>( + addAllTransactions + ); + const attemptedSavedTransactions = blockTransactions.unknown; + const savedTransactionCount = Number(saveTransactionsResult.rows[0]!.count); + const transactionCacheMisses = + attemptedSavedTransactions.length - savedTransactionCount; + const addBlockResult = await client.query<{ + insertedBlockTransactionCount: string; + insertedNodeBlockCount: string; + joinedTransactionCount: string; + }>(addBlockQuery); + await verifyBlockTransactionsLinked( + client, + block, + Number(addBlockResult.rows[0]!.joinedTransactionCount) + ); + await client.query('COMMIT;'); + return { + attemptedSavedTransactions, + transactionCacheMisses, + }; + } catch (err) { + await client.query('ROLLBACK;'); + // eslint-disable-next-line functional/no-throw-statement + throw err; + } finally { + client.release(); + } }; /** @@ -564,17 +1774,33 @@ DELETE FROM node_block WHERE /** * After initial sync, Chaingraph begins tracking each node's mempool. * - * To maintain consistency, a trigger which is disabled before initial sync must + * To maintain consistency, triggers which are disabled before initial sync must * be reenabled to clear any confirmed or conflicting transactions when a block * is accepted. */ export const reenableMempoolCleaning = async () => { const client = await pool.connect(); - const res = await client.query( + await client.query( `ALTER TABLE node_block ENABLE TRIGGER trigger_public_node_block_insert;` ); + const triggerExists = + // cspell:ignore tgrelid tgname + ( + await client.query<{ triggerExists: boolean }>(/* sql */ ` +SELECT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgrelid = 'node_transaction_history'::regclass + AND tgname = 'trigger_public_node_transaction_history_insert' +) AS "triggerExists"; +`) + ).rows[0]?.triggerExists === true; + if (triggerExists) { + await client.query( + `ALTER TABLE node_transaction_history ENABLE TRIGGER trigger_public_node_transaction_history_insert;` + ); + } client.release(); - return res.rowCount; + return triggerExists; }; /** diff --git a/src/e2e/e2e.spec.ts b/src/e2e/e2e.spec.ts index 8d62a424..2a2aafad 100644 --- a/src/e2e/e2e.spec.ts +++ b/src/e2e/e2e.spec.ts @@ -27,6 +27,10 @@ import { execa } from 'execa'; import got from 'got'; import pg from 'pg'; +import { indexDefinitions } from '../components/db-utils.js'; +import type * as DbModule from '../db.js'; +import type { ChaingraphTransaction } from '../types/chaingraph.js'; + import { chaingraphE2eLogPath, logger } from './e2e.spec.logging.helper.js'; import { chipnetCashTokensTx, @@ -59,10 +63,32 @@ const recreateDbOnStartup = true as boolean; const dir = dirname(fileURLToPath(import.meta.url)); const migration = (path: string) => resolve(dir, '../../images/hasura/hasura-data/migrations/', path); +const backfillOrphanMempoolDescendantsMigrationPath = migration( + 'default/1778158619747_backfill_orphan_mempool_descendants/up.sql' +); const dbUpMigrationPaths = [ migration('default/1616195337538_init/up.sql'), migration('default/1673124945608_tokens/up.sql'), migration('default/1676794104752_parse_bytecode_pattern/up.sql'), + migration( + 'default/1778151011521_cascade_invalidate_mempool_descendants/up.sql' + ), + backfillOrphanMempoolDescendantsMigrationPath, + migration('default/1778351597200_fix_block_encoded_transaction_count/up.sql'), + migration( + 'default/1778415174939_fix_data_carrier_outputs_empty_bytecode/up.sql' + ), + migration('default/1778429124205_fix_coinbase_only_value_aggregates/up.sql'), + migration( + 'default/1778435997270_add_node_transaction_history_primary_key/up.sql' + ), + migration('default/1778437612917_fix_zero_length_pushdata_patterns/up.sql'), + migration('default/1778438318512_fix_redeem_bytecode_parser/up.sql'), + migration('default/1791100000000_fix_search_output_exact_matching/up.sql'), + migration( + 'default/1791100001000_fix_search_output_prefix_literal_bytes/up.sql' + ), + migration('default/1791200000000_add_incomplete_block_audit/up.sql'), ]; const chaingraphInternalApiPort = '3201'; @@ -138,9 +164,12 @@ const postgresE2eConnectionStringTestDb = `${postgresE2eConnectionStringBase}/${ const e2eEnvVariables = { /* eslint-disable @typescript-eslint/naming-convention */ CHAINGRAPH_GENESIS_BLOCKS: `${e2eTestNetworkMagicHex}:${genesisBlockRaw},e3e1f3e8:${genesisBlockRaw},dab5bffa:${testnetGenesisBlockRaw}`, + CHAINGRAPH_INCOMPLETE_BLOCK_REPAIR_BATCH_SIZE: '10000', CHAINGRAPH_INTERNAL_API_PORT: chaingraphInternalApiPort, CHAINGRAPH_LOG_FIREHOSE: logP2pMessage.toString(), CHAINGRAPH_LOG_PATH: chaingraphE2eLogPath, + CHAINGRAPH_MEMPOOL_RESYNC_MIN_INTERVAL_MS: '100', + CHAINGRAPH_MEMPOOL_TRANSACTION_EXPIRATION_SCAN_INTERVAL_MS: '100', CHAINGRAPH_POSTGRES_CONNECTION_STRING: postgresE2eConnectionStringTestDb, CHAINGRAPH_TRUSTED_NODES: e2eTrustedNodesSet1, NODE_ENV: 'production', @@ -165,6 +194,40 @@ const peers = { node3: placeholder, }; +/** + * The transaction hashes (hex, as displayed) each mock node announces in reply + * to a BIP35 `mempool` request, and the number of requests each has received. + */ +const mempoolReplies: { [nodeName in keyof typeof peers]: string[] } = { + node1: [], + node2: [], + node3: [], +}; +const mempoolRequestCounts: { [nodeName in keyof typeof peers]: number } = { + node1: 0, + node2: 0, + node3: 0, +}; +const respondToMempoolRequests = (nodeName: keyof typeof peers, peer: Peer) => { + peer.on('mempool', () => { + mempoolRequestCounts[nodeName] += 1; + logger.debug( + `e2e: ${nodeName} received mempool request, announcing: ${mempoolReplies[ + nodeName + ].join(', ')}` + ); + /** + * BIP35 replies may be split across multiple `inv` messages; announce each + * transaction in its own message. + */ + mempoolReplies[nodeName].forEach((hash) => { + peer.sendMessage( + peer.messages.Inventory.forTransaction(Buffer.from(hash, 'hex')) + ); + }); + }); +}; + test.beforeEach((t) => { logger.debug(`Starting test: ${t.title}`); }); @@ -217,15 +280,24 @@ test.before(async () => { }; node1.on('peerready', (peer) => { logPeerConnection(`node1`, peer); - if (!peer.subversion.includes('tx-broadcast')) peers.node1 = peer; + if (!peer.subversion.includes('tx-broadcast')) { + peers.node1 = peer; + respondToMempoolRequests('node1', peer); + } }); node2.on('peerready', (peer) => { logPeerConnection(`node2`, peer); - if (!peer.subversion.includes('tx-broadcast')) peers.node2 = peer; + if (!peer.subversion.includes('tx-broadcast')) { + peers.node2 = peer; + respondToMempoolRequests('node2', peer); + } }); node3.on('peerready', (peer) => { logPeerConnection(`node3`, peer); - if (!peer.subversion.includes('tx-broadcast')) peers.node3 = peer; + if (!peer.subversion.includes('tx-broadcast')) { + peers.node3 = peer; + respondToMempoolRequests('node3', peer); + } }); if (logP2pMessage) { @@ -558,6 +630,257 @@ const sleep = async (ms: number) => setTimeout(res, ms); }); +const repeatedHashByteLength = 32; +const transactionSaveConflictPollingAttempts = 50; +const transactionSaveConflictPollingIntervalMs = 20; +const waitForTransactionSaveConflict = async ( + transactionHash: string, + remainingAttempts = transactionSaveConflictPollingAttempts +): Promise => { + const result = await client.query<{ waiting: boolean }>( + /* sql */ ` + SELECT EXISTS ( + SELECT 1 + FROM pg_stat_activity + -- cspell:disable-next-line + WHERE datname = $1 + AND query LIKE '%INSERT INTO transaction%' + AND query LIKE $2 + AND wait_event_type IS NOT NULL + ) AS waiting; + `, + [e2eTestDbName, `%${transactionHash}%`] + ); + if (result.rows[0]!.waiting) { + return; + } + if (remainingAttempts === 0) { + // eslint-disable-next-line functional/no-throw-statement + throw new Error( + `Timed out waiting for saveTransactionForNodes conflict on ${transactionHash}.` + ); + } + await sleep(transactionSaveConflictPollingIntervalMs); + await waitForTransactionSaveConflict(transactionHash, remainingAttempts - 1); +}; + +const blockRepairPollingAttempts = 40; +const blockRepairPollingIntervalMs = 250; +const getBlockTransactionCount = async (blockHash: string) => + Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*) FROM block_transaction + INNER JOIN block ON block.internal_id = block_transaction.block_internal_id + WHERE block.hash = $1; + `, + [hexToBin(blockHash)] + ) + ).rows[0]!.count + ); +const waitForBlockTransactionCount = async ( + blockHash: string, + expectedCount: number, + remainingAttempts = blockRepairPollingAttempts +): Promise => { + const count = await getBlockTransactionCount(blockHash); + if (count === expectedCount || remainingAttempts === 0) { + return count; + } + await sleep(blockRepairPollingIntervalMs); + return waitForBlockTransactionCount( + blockHash, + expectedCount, + remainingAttempts - 1 + ); +}; + +/** + * When several mock nodes announce the same block, the agent's "Saved new + * block" log line lists only the nodes whose headers arrived before the save; + * later announcers are recorded via "accepted existing blocks" instead. Header + * arrival order across peer connections isn't deterministic, so tests check + * the resulting node_block rows rather than the log line. + */ +const blockAcceptancePollingAttempts = 100; +const blockAcceptancePollingIntervalMs = 100; +const getAcceptingNodeNames = async (blockHash: string) => + ( + await client.query<{ nodeName: string }>( + /* sql */ ` + SELECT node.name AS "nodeName" + FROM node_block + INNER JOIN node ON node.internal_id = node_block.node_internal_id + INNER JOIN block ON block.internal_id = node_block.block_internal_id + WHERE block.hash = $1 + ORDER BY node.name; + `, + [hexToBin(blockHash)] + ) + ).rows.map((row) => row.nodeName); +const waitForBlockAcceptedByNodes = async ( + blockHash: string, + expectedNodeNames: string[], + remainingAttempts = blockAcceptancePollingAttempts +): Promise => { + const acceptingNodeNames = await getAcceptingNodeNames(blockHash); + if ( + expectedNodeNames.every((nodeName) => + acceptingNodeNames.includes(nodeName) + ) || + remainingAttempts === 0 + ) { + return acceptingNodeNames; + } + await sleep(blockAcceptancePollingIntervalMs); + return waitForBlockAcceptedByNodes( + blockHash, + expectedNodeNames, + remainingAttempts - 1 + ); +}; + +const mempoolExpirationPollingAttempts = 50; +const mempoolExpirationPollingIntervalMs = 100; +const getExpiredMempoolArchiveState = async () => + ( + await client.query<{ + historyRowCount: number; + inMempool: boolean; + replacedAt: string | null; + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')), + ('expiry_child_c', decode(repeat('d3', 32), 'hex')) +), +selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +SELECT named_transactions.name AS "transactionName", + (node_transaction.transaction_internal_id IS NOT NULL) AS "inMempool", + COUNT(node_transaction_history.transaction_internal_id)::integer AS "historyRowCount", + MIN(node_transaction_history.replaced_at)::text AS "replacedAt" + FROM named_transactions + CROSS JOIN selected_node + LEFT JOIN node_transaction + ON node_transaction.node_internal_id = selected_node.internal_id + AND node_transaction.transaction_internal_id = named_transactions.internal_id + LEFT JOIN node_transaction_history + ON node_transaction_history.node_internal_id = selected_node.internal_id + AND node_transaction_history.transaction_internal_id = named_transactions.internal_id + GROUP BY named_transactions.name, node_transaction.transaction_internal_id + ORDER BY named_transactions.name; +`) + ).rows; +const waitForExpiredMempoolArchive = async ( + remainingAttempts = mempoolExpirationPollingAttempts +): Promise>> => { + const rows = await getExpiredMempoolArchiveState(); + const expectedReplacedAt = '2026-01-15 00:00:00'; + if ( + rows.every( + (row) => + !row.inMempool && + row.historyRowCount === 1 && + row.replacedAt === expectedReplacedAt + ) || + remainingAttempts === 0 + ) { + return rows; + } + await sleep(mempoolExpirationPollingIntervalMs); + return waitForExpiredMempoolArchive(remainingAttempts - 1); +}; + +const getConfirmedMempoolArchiveState = async () => + ( + await client.query<{ + historyRowCount: number; + inMempool: boolean; + replacedAt: string | null; + }>(/* sql */ ` +WITH selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +) +SELECT (node_transaction.transaction_internal_id IS NOT NULL) AS "inMempool", + COUNT(node_transaction_history.transaction_internal_id)::integer AS "historyRowCount", + MIN(node_transaction_history.replaced_at)::text AS "replacedAt" + FROM selected_transaction + CROSS JOIN selected_node + LEFT JOIN node_transaction + ON node_transaction.node_internal_id = selected_node.internal_id + AND node_transaction.transaction_internal_id = selected_transaction.internal_id + LEFT JOIN node_transaction_history + ON node_transaction_history.node_internal_id = selected_node.internal_id + AND node_transaction_history.transaction_internal_id = selected_transaction.internal_id + GROUP BY node_transaction.transaction_internal_id; +`) + ).rows[0]; + +const confirmedMempoolArchiveCompleted = ( + row: Awaited> +) => + row !== undefined && + !row.inMempool && + row.historyRowCount === 1 && + row.replacedAt === null; + +const waitForConfirmedMempoolArchive = async ( + remainingAttempts = mempoolExpirationPollingAttempts +): Promise>> => { + const row = await getConfirmedMempoolArchiveState(); + if (confirmedMempoolArchiveCompleted(row) || remainingAttempts === 0) { + return row; + } + await sleep(mempoolExpirationPollingIntervalMs); + return waitForConfirmedMempoolArchive(remainingAttempts - 1); +}; + +const mempoolPollingAttempts = 50; +const mempoolPollingIntervalMs = 100; +const getMempoolTransactionHashes = async () => + ( + await client.query<{ hash: string }>( + /* sql */ `SELECT encode(hash, 'hex') AS hash FROM node_transaction JOIN transaction ON node_transaction.transaction_internal_id = transaction.internal_id ORDER BY hash ASC;` + ) + ).rows.map((row) => row.hash); +/** + * Poll node_transaction until its hashes (as listed by + * {@link getMempoolTransactionHashes}) satisfy `expected`: the agent saves + * transactions received from peers asynchronously. Returns the last hashes + * read. + */ +const waitForMempoolTransactionHashes = async ( + expected: (hashes: string[]) => boolean, + remainingAttempts = mempoolPollingAttempts +): Promise => { + const hashes = await getMempoolTransactionHashes(); + if (expected(hashes) || remainingAttempts === 0) { + return hashes; + } + await sleep(mempoolPollingIntervalMs); + return waitForMempoolTransactionHashes(expected, remainingAttempts - 1); +}; + test.serial( '[e2e] ignores inbound transactions before initial sync is complete', async (t) => { @@ -586,6 +909,27 @@ test.serial('[e2e] completes initial sync', async (t) => { t.pass(); }); +test.serial( + '[e2e] requests the mempool of each node once mempool tracking is enabled', + async (t) => { + await waitForStdout('Agent: enabled mempool tracking.'); + await waitForStdout( + 'node1: requested node mempool (BIP35) – triggers: mempool-tracking-enabled' + ); + await waitForStdout( + 'node2: requested node mempool (BIP35) – triggers: mempool-tracking-enabled' + ); + await waitForStdout( + 'node3: requested node mempool (BIP35) – triggers: mempool-tracking-enabled' + ); + const delay = 500; + await sleep(delay); + t.true(mempoolRequestCounts.node1 >= 1); + t.true(mempoolRequestCounts.node2 >= 1); + t.true(mempoolRequestCounts.node3 >= 1); + } +); + test.serial('[e2e] creates expected indexes after initial sync', async (t) => { await waitForStdout('Agent: all managed indexes have been created.'); await waitForStdout('Agent: enabled mempool tracking.'); @@ -603,23 +947,935 @@ test.serial('[e2e] creates expected indexes after initial sync', async (t) => { 'block_internal_id_key', 'block_pkey', 'block_transaction_pkey', + 'incomplete_block_audit_pkey', 'input_pkey', 'node_block_history_pkey', 'node_block_pkey', 'node_internal_id_key', 'node_name_key', 'node_pkey', + 'node_transaction_history_pkey', 'node_transaction_pkey', 'output_pkey', 'output_search_index', 'spent_by_index', + 'token_category_index', 'transaction_hash_key', 'transaction_pkey', ]); + // cspell:ignore tgenabled tgname + const triggers = ( + await client.query<{ + tgenabled: string; + tgname: string; + }>(/* sql */ ` + SELECT tgname, tgenabled FROM pg_trigger + WHERE tgname IN ( + 'trigger_public_node_block_insert', + 'trigger_public_node_transaction_history_insert' + ) + ORDER BY tgname; + `) + ).rows; + t.deepEqual(triggers, [ + { tgenabled: 'O', tgname: 'trigger_public_node_block_insert' }, + { + tgenabled: 'O', + tgname: 'trigger_public_node_transaction_history_insert', + }, + ]); clearStdoutBuffer(); t.pass(); }); +test.serial( + '[e2e] getAllKnownBlockHashes returns hex hashes for every known block', + async (t) => { + const originalPostgresConnectionString = + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + postgresE2eConnectionStringTestDb; + const { getAllKnownBlockHashes } = await import('../db.js'); + // eslint-disable-next-line functional/no-try-statement + try { + const hashes = await getAllKnownBlockHashes(); + /* + * Convert client-side (the previous implementation) to verify the + * SQL-side `encode(...)` used by `getAllKnownBlockHashes` matches it. + */ + const expected = ( + await client.query<{ hash: Buffer }>(/* sql */ ` + SELECT hash FROM block ORDER BY hash; + `) + ).rows.map((row) => row.hash.toString('hex')); + t.true(expected.length > 0); + t.deepEqual( + [...hashes].sort((a, b) => (a < b ? -1 : Number(a > b))), + expected + ); + const hexPattern = /^[0-9a-f]{64}$/u; + t.true(hashes.every((hash) => hexPattern.test(hash))); + } finally { + if (originalPostgresConnectionString === undefined) { + delete process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + } else { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + originalPostgresConnectionString; + } + } + } +); + +test.serial( + '[e2e] records node validation after concurrent transaction insert conflict', + async (t) => { + const transactionHash = 'c1'.repeat(repeatedHashByteLength); + const validatedAt = new Date('2026-01-01T00:00:00.000Z'); + const transaction: ChaingraphTransaction = { + hash: transactionHash, + inputs: [ + { + outpointIndex: 0, + outpointTransactionHash: 'c2'.repeat(repeatedHashByteLength), + sequenceNumber: 0, + unlockingBytecode: '51', + }, + ], + isCoinbase: false, + locktime: 0, + outputs: [ + { + lockingBytecode: '51', + valueSatoshis: 1000n, + }, + ], + sizeBytes: 100, + version: 1, + }; + const nodeInternalId = Number( + ( + await client.query<{ internalId: number }>( + /* sql */ `SELECT internal_id AS "internalId" FROM node WHERE name = 'node1';` + ) + ).rows[0]!.internalId + ); + await client.query( + /* sql */ ` + DELETE FROM node_transaction + USING transaction + WHERE node_transaction.transaction_internal_id = transaction.internal_id + AND transaction.hash = $1; + `, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query( + /* sql */ ` + DELETE FROM input + USING transaction + WHERE input.transaction_internal_id = transaction.internal_id + AND transaction.hash = $1; + `, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query( + /* sql */ `DELETE FROM output WHERE transaction_hash = $1;`, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query(/* sql */ `DELETE FROM transaction WHERE hash = $1;`, [ + Buffer.from(transactionHash, 'hex'), + ]); + const originalPostgresConnectionString = + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + postgresE2eConnectionStringTestDb; + const { pool: dbPool, saveTransactionForNodes } = await import('../db.js'); + const competingClient = new pg.Client({ + connectionString: postgresE2eConnectionStringTestDb, + }); + await competingClient.connect(); + // eslint-disable-next-line functional/no-let + let competingTransactionOpen = false; + // eslint-disable-next-line functional/no-try-statement + try { + await competingClient.query(/* sql */ `BEGIN;`); + competingTransactionOpen = true; + await competingClient.query( + /* sql */ ` + INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + VALUES ($1, 1, 0, 100, false); + `, + [Buffer.from(transactionHash, 'hex')] + ); + const savePromise = saveTransactionForNodes(transaction, [ + { nodeInternalId, validatedAt }, + ]); + await waitForTransactionSaveConflict(transactionHash); + await competingClient.query(/* sql */ `COMMIT;`); + competingTransactionOpen = false; + await t.notThrowsAsync(savePromise); + const savedValidationCount = Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*)::bigint AS count + FROM node_transaction + JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + WHERE transaction.hash = $1 + AND node_transaction.node_internal_id = $2 + AND node_transaction.validated_at = $3; + `, + [Buffer.from(transactionHash, 'hex'), nodeInternalId, validatedAt] + ) + ).rows[0]!.count + ); + t.deepEqual(savedValidationCount, 1); + } finally { + if (competingTransactionOpen) { + await competingClient.query(/* sql */ `ROLLBACK;`).catch((err) => { + logger.debug(err); + }); + } + await client.query( + /* sql */ ` + DELETE FROM node_transaction + USING transaction + WHERE node_transaction.transaction_internal_id = transaction.internal_id + AND transaction.hash = $1; + `, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query( + /* sql */ ` + DELETE FROM input + USING transaction + WHERE input.transaction_internal_id = transaction.internal_id + AND transaction.hash = $1; + `, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query( + /* sql */ `DELETE FROM output WHERE transaction_hash = $1;`, + [Buffer.from(transactionHash, 'hex')] + ); + await client.query(/* sql */ `DELETE FROM transaction WHERE hash = $1;`, [ + Buffer.from(transactionHash, 'hex'), + ]); + await competingClient.end(); + await dbPool.end(); + if (originalPostgresConnectionString === undefined) { + delete process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + } else { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + originalPostgresConnectionString; + } + } + } +); + +test.serial( + '[e2e] cascades replaced mempool transaction history to same-node descendants', + async (t) => { + await client.query(/* sql */ `BEGIN;`); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('parent_a', decode(repeat('f1', 32), 'hex')), + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +) +INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, 1, 0, 100, false + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('parent_a', decode(repeat('f1', 32), 'hex')), + ('child_b', decode(repeat('f3', 32), 'hex')) +) +INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + SELECT hash, 0, 1000, '\\x51'::bytea + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH input_values (child_name, parent_name, input_index) AS ( + VALUES + ('child_b', 'parent_a', 0), + ('child_c', 'child_b', 0) +), +transaction_values (name, hash) AS ( + VALUES + ('parent_a', decode(repeat('f1', 32), 'hex')), + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id, transaction.hash + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT child.internal_id, input_values.input_index, 0, 0, parent.hash, '\\x51'::bytea + FROM input_values + JOIN named_transactions child + ON child.name = input_values.child_name + JOIN named_transactions parent + ON parent.name = input_values.parent_name; +`); + await client.query(/* sql */ ` +WITH selected_nodes AS ( + SELECT name, internal_id + FROM node + WHERE name IN ('node1', 'node2') +), +transaction_values (name, hash) AS ( + VALUES + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT selected_nodes.internal_id, named_transactions.internal_id, timestamp '2026-01-01 00:00:00' + FROM selected_nodes + CROSS JOIN named_transactions; +`); + await client.query(/* sql */ ` +WITH selected_nodes AS ( + SELECT name, internal_id + FROM node + WHERE name = 'node1' +), +transaction_values (name, hash) AS ( + VALUES + ('parent_a', decode(repeat('f1', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT selected_nodes.internal_id, + named_transactions.internal_id, + timestamp '2026-01-01 00:00:00', + timestamp '2026-01-01 00:10:00' + FROM selected_nodes + CROSS JOIN named_transactions; +`); + const remainingMempool = ( + await client.query<{ + nodeName: string; + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +) +SELECT node.name AS "nodeName", transaction_values.name AS "transactionName" + FROM node_transaction + JOIN node + ON node.internal_id = node_transaction.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + JOIN transaction_values + ON transaction_values.hash = transaction.hash + ORDER BY "nodeName", "transactionName"; +`) + ).rows; + t.deepEqual(remainingMempool, [ + { nodeName: 'node2', transactionName: 'child_b' }, + { nodeName: 'node2', transactionName: 'child_c' }, + ]); + const archivedDescendants = ( + await client.query<{ + replacedAt: string; + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('child_b', decode(repeat('f3', 32), 'hex')), + ('child_c', decode(repeat('f4', 32), 'hex')) +) +SELECT transaction_values.name AS "transactionName", + node_transaction_history.replaced_at::text AS "replacedAt" + FROM node_transaction_history + JOIN node + ON node.internal_id = node_transaction_history.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction_history.transaction_internal_id + JOIN transaction_values + ON transaction_values.hash = transaction.hash + WHERE node.name = 'node1' + ORDER BY "transactionName"; +`) + ).rows; + t.deepEqual(archivedDescendants, [ + { replacedAt: '2026-01-01 00:10:00', transactionName: 'child_b' }, + { replacedAt: '2026-01-01 00:10:00', transactionName: 'child_c' }, + ]); + } finally { + await client.query(/* sql */ `ROLLBACK;`); + } + } +); + +test.serial( + '[e2e] archives expired mempool transactions and descendants', + async (t) => { + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')), + ('expiry_child_c', decode(repeat('d3', 32), 'hex')) +) +INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, 1, 0, 100, false + FROM transaction_values; +`); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')) +) +INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + SELECT hash, 0, 1000, '\\x51'::bytea + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH input_values (child_name, parent_name, input_index) AS ( + VALUES + ('expiry_child_b', 'expiry_parent_a', 0), + ('expiry_child_c', 'expiry_child_b', 0) +), +transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')), + ('expiry_child_c', decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id, transaction.hash + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT child.internal_id, input_values.input_index, 0, 0, parent.hash, '\\x51'::bytea + FROM input_values + JOIN named_transactions child + ON child.name = input_values.child_name + JOIN named_transactions parent + ON parent.name = input_values.parent_name; +`); + await client.query(/* sql */ ` +WITH selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +transaction_values (name, hash) AS ( + VALUES + ('expiry_parent_a', decode(repeat('d1', 32), 'hex')), + ('expiry_child_b', decode(repeat('d2', 32), 'hex')), + ('expiry_child_c', decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT selected_node.internal_id, + named_transactions.internal_id, + CASE + WHEN named_transactions.name = 'expiry_parent_a' + THEN timestamp '2026-01-01 00:00:00' + ELSE CURRENT_TIMESTAMP + interval '1 day' + END + FROM selected_node + CROSS JOIN named_transactions; +`); + const archivedTransactions = await waitForExpiredMempoolArchive(); + t.deepEqual(archivedTransactions, [ + { + historyRowCount: 1, + inMempool: false, + replacedAt: '2026-01-15 00:00:00', + transactionName: 'expiry_child_b', + }, + { + historyRowCount: 1, + inMempool: false, + replacedAt: '2026-01-15 00:00:00', + transactionName: 'expiry_child_c', + }, + { + historyRowCount: 1, + inMempool: false, + replacedAt: '2026-01-15 00:00:00', + transactionName: 'expiry_parent_a', + }, + ]); + } finally { + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM node_transaction + USING named_transactions + WHERE node_transaction.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM node_transaction_history + USING named_transactions + WHERE node_transaction_history.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM input + USING named_transactions + WHERE input.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +) +DELETE FROM output + USING transaction_values + WHERE output.transaction_hash = transaction_values.hash; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d1', 32), 'hex')), + (decode(repeat('d2', 32), 'hex')), + (decode(repeat('d3', 32), 'hex')) +) +DELETE FROM transaction + USING transaction_values + WHERE transaction.hash = transaction_values.hash; +`); + } + } +); + +test.serial( + '[e2e] archives stale mempool transactions already accepted by blocks', + async (t) => { + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('confirmed_parent_a', decode(repeat('d4', 32), 'hex')), + ('confirmed_child_b', decode(repeat('d5', 32), 'hex')) +) +INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, 1, 0, 100, false + FROM transaction_values; +`); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query(/* sql */ ` +INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + VALUES (decode(repeat('d4', 32), 'hex'), 0, 1000, '\\x51'::bytea); +`); + await client.query(/* sql */ ` +WITH selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +) +INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT selected_transaction.internal_id, + 0, + 0, + 0, + decode(repeat('d4', 32), 'hex'), + '\\x51'::bytea + FROM selected_transaction; +`); + await client.query(/* sql */ ` +WITH selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +), +inserted_block AS ( + INSERT INTO block (height, version, "timestamp", hash, previous_block_hash, merkle_root, bits, nonce, size_bytes) + VALUES (4001, 1, 0, decode(repeat('d6', 32), 'hex'), decode(repeat('d7', 32), 'hex'), decode(repeat('d8', 32), 'hex'), 0, 0, 181) + RETURNING internal_id +) +INSERT INTO block_transaction (block_internal_id, transaction_internal_id, transaction_index) + SELECT inserted_block.internal_id, selected_transaction.internal_id, 1 + FROM inserted_block + CROSS JOIN selected_transaction; +`); + await client.query(/* sql */ ` +WITH selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +selected_block AS ( + SELECT internal_id + FROM block + WHERE hash = decode(repeat('d6', 32), 'hex') +) +INSERT INTO node_block (node_internal_id, block_internal_id, accepted_at) + SELECT selected_node.internal_id, + selected_block.internal_id, + timestamp '2026-01-01 00:10:00' + FROM selected_node + CROSS JOIN selected_block; +`); + await client.query(/* sql */ ` +WITH selected_node AS ( + SELECT internal_id + FROM node + WHERE name = 'node1' +), +selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +) +INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT selected_node.internal_id, + selected_transaction.internal_id, + timestamp '2026-01-01 00:00:00' + FROM selected_node + CROSS JOIN selected_transaction; +`); + const archivedTransaction = await waitForConfirmedMempoolArchive(); + t.deepEqual(archivedTransaction, { + historyRowCount: 1, + inMempool: false, + replacedAt: null, + }); + } finally { + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d4', 32), 'hex')), + (decode(repeat('d5', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM node_transaction + USING named_transactions + WHERE node_transaction.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d4', 32), 'hex')), + (decode(repeat('d5', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +DELETE FROM node_transaction_history + USING named_transactions + WHERE node_transaction_history.transaction_internal_id = named_transactions.internal_id; +`); + await client.query(/* sql */ ` +WITH selected_block AS ( + SELECT internal_id + FROM block + WHERE hash = decode(repeat('d6', 32), 'hex') +) +DELETE FROM node_block + USING selected_block + WHERE node_block.block_internal_id = selected_block.internal_id; +`); + await client.query(/* sql */ ` +WITH selected_block AS ( + SELECT internal_id + FROM block + WHERE hash = decode(repeat('d6', 32), 'hex') +) +DELETE FROM node_block_history + USING selected_block + WHERE node_block_history.block_internal_id = selected_block.internal_id; +`); + await client.query(/* sql */ ` +WITH selected_block AS ( + SELECT internal_id + FROM block + WHERE hash = decode(repeat('d6', 32), 'hex') +) +DELETE FROM block_transaction + USING selected_block + WHERE block_transaction.block_internal_id = selected_block.internal_id; +`); + await client.query(/* sql */ ` +DELETE FROM block + WHERE hash = decode(repeat('d6', 32), 'hex'); +`); + await client.query(/* sql */ ` +WITH selected_transaction AS ( + SELECT internal_id + FROM transaction + WHERE hash = decode(repeat('d5', 32), 'hex') +) +DELETE FROM input + USING selected_transaction + WHERE input.transaction_internal_id = selected_transaction.internal_id; +`); + await client.query(/* sql */ ` +DELETE FROM output + WHERE transaction_hash = decode(repeat('d4', 32), 'hex'); +`); + await client.query(/* sql */ ` +WITH transaction_values (hash) AS ( + VALUES + (decode(repeat('d4', 32), 'hex')), + (decode(repeat('d5', 32), 'hex')) +) +DELETE FROM transaction + USING transaction_values + WHERE transaction.hash = transaction_values.hash; +`); + } + } +); + +test.serial( + '[e2e] backfills existing orphan mempool descendants with idempotence', + async (t) => { + await client.query(/* sql */ `BEGIN;`); + // eslint-disable-next-line functional/no-try-statement + try { + const backfillMigration = readFileSync( + backfillOrphanMempoolDescendantsMigrationPath, + 'utf8' + ); + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')), + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +) +INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + SELECT hash, 1, 0, 100, false + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')), + ('backfill_child_b', decode(repeat('e2', 32), 'hex')) +) +INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + SELECT hash, 0, 1000, '\\x51'::bytea + FROM transaction_values; +`); + await client.query(/* sql */ ` +WITH input_values (child_name, parent_name, input_index) AS ( + VALUES + ('backfill_child_b', 'backfill_parent_a', 0), + ('backfill_child_c', 'backfill_child_b', 0) +), +transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')), + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id, transaction.hash + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO input (transaction_internal_id, input_index, outpoint_index, sequence_number, outpoint_transaction_hash, unlocking_bytecode) + SELECT child.internal_id, input_values.input_index, 0, 0, parent.hash, '\\x51'::bytea + FROM input_values + JOIN named_transactions child + ON child.name = input_values.child_name + JOIN named_transactions parent + ON parent.name = input_values.parent_name; +`); + await client.query(/* sql */ ` +WITH selected_nodes AS ( + SELECT name, internal_id + FROM node + WHERE name = 'node1' +), +transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction_history (node_internal_id, transaction_internal_id, validated_at, replaced_at) + SELECT selected_nodes.internal_id, + named_transactions.internal_id, + timestamp '2026-01-01 00:00:00', + timestamp '2026-01-01 00:10:00' + FROM selected_nodes + CROSS JOIN named_transactions; +`); + await client.query(/* sql */ ` +WITH selected_nodes AS ( + SELECT name, internal_id + FROM node + WHERE name = 'node1' +), +transaction_values (name, hash) AS ( + VALUES + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +), +named_transactions AS ( + SELECT transaction_values.name, transaction.internal_id + FROM transaction + JOIN transaction_values + ON transaction_values.hash = transaction.hash +) +INSERT INTO node_transaction (node_internal_id, transaction_internal_id, validated_at) + SELECT selected_nodes.internal_id, named_transactions.internal_id, timestamp '2026-01-01 00:00:00' + FROM selected_nodes + CROSS JOIN named_transactions; +`); + await client.query(backfillMigration); + await client.query(backfillMigration); + + const remainingMempool = ( + await client.query<{ + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +) +SELECT transaction_values.name AS "transactionName" + FROM node_transaction + JOIN node + ON node.internal_id = node_transaction.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + JOIN transaction_values + ON transaction_values.hash = transaction.hash + WHERE node.name = 'node1' + ORDER BY "transactionName"; +`) + ).rows; + t.deepEqual(remainingMempool, []); + + const archivedTransactions = ( + await client.query<{ + historyRowCount: number; + replacedAt: string; + transactionName: string; + }>(/* sql */ ` +WITH transaction_values (name, hash) AS ( + VALUES + ('backfill_parent_a', decode(repeat('e1', 32), 'hex')), + ('backfill_child_b', decode(repeat('e2', 32), 'hex')), + ('backfill_child_c', decode(repeat('e3', 32), 'hex')) +) +SELECT transaction_values.name AS "transactionName", + COUNT(*)::integer AS "historyRowCount", + MIN(node_transaction_history.replaced_at)::text AS "replacedAt" + FROM node_transaction_history + JOIN node + ON node.internal_id = node_transaction_history.node_internal_id + JOIN transaction + ON transaction.internal_id = node_transaction_history.transaction_internal_id + JOIN transaction_values + ON transaction_values.hash = transaction.hash + WHERE node.name = 'node1' + GROUP BY transaction_values.name + ORDER BY "transactionName"; +`) + ).rows; + t.deepEqual(archivedTransactions, [ + { + historyRowCount: 1, + replacedAt: '2026-01-01 00:10:00', + transactionName: 'backfill_child_b', + }, + { + historyRowCount: 1, + replacedAt: '2026-01-01 00:10:00', + transactionName: 'backfill_child_c', + }, + { + historyRowCount: 1, + replacedAt: '2026-01-01 00:10:00', + transactionName: 'backfill_parent_a', + }, + ]); + } finally { + await client.query(/* sql */ `ROLLBACK;`); + } + } +); + test.serial( '[e2e] after initial sync is complete, requests transactions as they are announced', async (t) => { @@ -654,6 +1910,45 @@ test.serial( } ); +test.serial( + '[e2e] records validation when another node announces a known transaction', + async (t) => { + peers.node2.sendMessage( + new peers.node2.messages.Transaction(new Transaction(halTxRaw)) + ); + const delay = 1000; + await sleep(delay); + const validations = await client.query<{ name: string }>( + /* sql */ ` + SELECT node.name + FROM node_transaction + INNER JOIN node + ON node.internal_id = node_transaction.node_internal_id + INNER JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + WHERE transaction.hash = $1 + ORDER BY node.name ASC; + `, + [hexToBin(halTxHash)] + ); + t.deepEqual( + validations.rows.map(({ name }) => name), + ['node1', 'node2'] + ); + await client.query( + /* sql */ ` + DELETE FROM node_transaction + USING node, transaction + WHERE node_transaction.node_internal_id = node.internal_id + AND node_transaction.transaction_internal_id = transaction.internal_id + AND node.name = 'node2' + AND transaction.hash = $1; + `, + [hexToBin(halTxHash)] + ); + } +); + test.serial('[e2e] handles first chipnet CashTokens transaction', async (t) => { peers.node1.sendMessage( new peers.node1.messages.Transaction(new Transaction(chipnetCashTokensTx)) @@ -733,6 +2028,82 @@ test.serial( } ); +test.serial( + '[e2e] [postgres] value aggregates handle coinbase-only blocks', + async (t) => { + const aggregates = ( + await client.query<{ + feeSatoshis: string; + generatedValueSatoshis: string; + inputValueSatoshis: string; + outputValueSatoshis: string; + }>(/* sql */ ` + SELECT + block_fee_satoshis(block)::text AS "feeSatoshis", + block_generated_value_satoshis(block)::text AS "generatedValueSatoshis", + block_input_value_satoshis(block)::text AS "inputValueSatoshis", + block_output_value_satoshis(block)::text AS "outputValueSatoshis" + FROM block WHERE height = 0; + `) + ).rows[0]!; + t.deepEqual(aggregates, { + feeSatoshis: '0', + generatedValueSatoshis: '5000000000', + inputValueSatoshis: '0', + outputValueSatoshis: '5000000000', + }); + } +); + +test.serial( + '[e2e] get hex-encoded block with multiple transactions', + async (t) => { + const blockWithMultipleTransactions = mockchainBeforeFork[1]!; + t.true(blockWithMultipleTransactions.transactions.length > 1); + /* eslint-disable @typescript-eslint/naming-convention */ + const encodedHex = ( + await client.query<{ block_encoded_hex: string }>( + /* sql */ `SELECT block_encoded_hex(block) FROM block WHERE hash = $1::bytea;`, + [hexToBin(blockWithMultipleTransactions.header.hash)] + ) + ).rows[0]!.block_encoded_hex; + /* eslint-enable @typescript-eslint/naming-convention */ + t.deepEqual(encodedHex, binToHex(blockWithMultipleTransactions.toBuffer())); + } +); + +test.serial( + '[e2e] [postgres] transaction_data_carrier_outputs ignores empty locking bytecode', + async (t) => { + const txHash = + '0000000000000000000000000000000000000000000000000000000000000075'; + await client.query( + /* sql */ ` + INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) + VALUES ($1::bytea, 1, 0, 10, false); + `, + [hexToBin(txHash)] + ); + await client.query( + /* sql */ ` + INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) + VALUES ($1::bytea, 0, 1, $2::bytea); + `, + [hexToBin(txHash), hexToBin('')] + ); + const outputs = await client.query<{ outputIndex: string }>( + /* sql */ ` + SELECT output_index AS "outputIndex" + FROM transaction_data_carrier_outputs( + (SELECT transaction FROM transaction WHERE hash = $1::bytea) + ); + `, + [hexToBin(txHash)] + ); + t.deepEqual(outputs.rows, []); + } +); + const newBlocks = ( node: 'node1' | 'node2' | 'node3', blocks: BitcoreBlock[] @@ -743,12 +2114,51 @@ const newBlocks = ( ); }; +test.serial( + '[e2e] survives Postgres terminating its connections (e.g. a restart)', + async (t) => { + const crashGracePeriodMs = 2_000; + const logStart = stdoutBuffer.length; + const terminated = await client.query<{ terminated: boolean }>( + /* sql */ ` + SELECT pg_terminate_backend(pid) AS terminated FROM pg_stat_activity + -- cspell:disable-next-line + WHERE datname = $1 + AND pid <> pg_backend_pid(); + `, + [e2eTestDbName] + ); + t.true( + terminated.rows.some((row) => row.terminated), + 'Expected the agent to hold at least one Postgres connection.' + ); + await waitForStdout('Postgres connection lost'); + // give a crash (uncaught 'error' event → shutdown) time to happen + await sleep(crashGracePeriodMs); + const logsSinceTermination = stdoutBuffer.slice(logStart); + t.false(logsSinceTermination.includes('uncaughtException')); + t.false(logsSinceTermination.includes('Shutting down...')); + const healthCheckResponse = await got( + `http://localhost:${chaingraphInternalApiPort}/health-check` + ); + t.deepEqual(healthCheckResponse.statusCode, StatusCode.success); + } +); + test.serial( '[e2e] syncs blocks as they arrive, handles multiple chain tips', async (t) => { const [, tx1] = tipA[0]!.transactions; peers.node1.sendMessage(new peers.node1.messages.Transaction(tx1)); logger.debug(`node1: sent tipA[0] transaction 0: ${tx1!.hash}`); + // the block must arrive after the transaction is saved for it to be known + t.true( + ( + await waitForMempoolTransactionHashes((hashes) => + hashes.includes(tx1!.hash) + ) + ).includes(tx1!.hash) + ); newBlocks('node1', [tipA[0]!]); newBlocks('node2', [tipB[0]!]); newBlocks('node3', [tipB[0]!]); @@ -763,10 +2173,80 @@ test.serial( ), '3 of 4 transactions should be new in tip A block 3001. Has the mockchain changed? (If so, update this test.)' ); - await waitForStdout( - /Saved new block – height:\s+3001[^\n]+nodes: node2, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipB[0]!.header.hash, [ + 'node2', + 'node3', + ]), + ['node2', 'node3'] ); - t.pass(); + } +); + +const getNodeValidationCount = async (nodeName: string, hash: string) => + Number( + ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*) FROM node_transaction + INNER JOIN node + ON node.internal_id = node_transaction.node_internal_id + INNER JOIN transaction + ON transaction.internal_id = node_transaction.transaction_internal_id + WHERE node.name = $1 AND transaction.hash = $2; + `, + [nodeName, hexToBin(hash)] + ) + ).rows[0]!.count + ); +const deleteNodeValidation = async (nodeName: string, hash: string) => + client.query( + /* sql */ ` + DELETE FROM node_transaction + USING node, transaction + WHERE node_transaction.node_internal_id = node.internal_id + AND node_transaction.transaction_internal_id = transaction.internal_id + AND node.name = $1 + AND transaction.hash = $2; + `, + [nodeName, hexToBin(hash)] + ); +const nodeValidationPollingAttempts = 50; +const nodeValidationPollingIntervalMs = 100; +/** + * Wait for the node's `node_transaction` row for the transaction, returning + * the final count of matching rows (`0` if it never appeared). + */ +const waitForNodeValidation = async ( + nodeName: string, + hash: string, + remainingAttempts = nodeValidationPollingAttempts +): Promise => { + const count = await getNodeValidationCount(nodeName, hash); + if (count !== 0 || remainingAttempts === 0) { + return count; + } + await sleep(nodeValidationPollingIntervalMs); + return waitForNodeValidation(nodeName, hash, remainingAttempts - 1); +}; + +/** + * Model https://github.com/bitauth/chaingraph/issues/81: node3 accepted a + * transaction (its `node_transaction` row was saved and the agent cached + * node3's acknowledgement), the transaction was later confirmed (archiving the + * row), then a re-organization returned it to node3's mempool. Nodes don't + * re-announce such transactions; they only appear in replies to `mempool`. + */ +test.serial( + '[e2e] prepares a transaction for return to the mempool by a re-org', + async (t) => { + peers.node3.sendMessage( + new peers.node3.messages.Transaction(new Transaction(halTxRaw)) + ); + t.is(await waitForNodeValidation('node3', halTxHash), 1); + await deleteNodeValidation('node3', halTxHash); + t.is(await getNodeValidationCount('node3', halTxHash), 0); + mempoolReplies.node3 = [halTxHash]; } ); @@ -785,10 +2265,53 @@ test.serial('[e2e] handles re-org of a single block', async (t) => { await waitForStdout( /Saved new block – height:\s+3002[^\n]+hash: a44a664d5acc560305fceb9ba3c7f195a5d78236c1705d5ae7434ba784005689[^\n]+nodes: node2/u ); - await waitForStdout( - /Saved new block – height:\s+3002[^\n]+hash: 9c4feec6f35a54f2244f2ab14e1370e60713097be2ead8402e4ef68f96e07c8c[^\n]+nodes: node1, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipA[1]!.header.hash, ['node1', 'node3']), + ['node1', 'node3'] ); - t.pass(); +}); + +test.serial( + '[e2e] re-records transactions returned to the mempool by a re-org', + async (t) => { + await waitForStdout( + 'node3: requested node mempool (BIP35) – triggers: reorganization' + ); + t.is(await waitForNodeValidation('node3', halTxHash), 1); + /** + * Clean up: stop announcing the transaction, wait for any pending replies, + * then remove node3's validation so later tests are unaffected. + */ + mempoolReplies.node3 = []; + const delay = 500; + await sleep(delay); + await deleteNodeValidation('node3', halTxHash); + } +); + +test.serial('[e2e] new block saved after reorg', async (t) => { + const acceptedBlocks = ( + await client.query<{ + hash: string; + nodeName: string; + }>( + /* sql */ ` + SELECT node.name AS "nodeName", encode(block.hash, 'hex') AS hash + FROM node_block + INNER JOIN node + ON node.internal_id = node_block.node_internal_id + INNER JOIN block + ON block.internal_id = node_block.block_internal_id + WHERE node.name = 'node3' + AND block.height = $1 + ORDER BY block.hash; + `, + [splitHeight + 1] + ) + ).rows; + t.deepEqual(acceptedBlocks, [ + { hash: tipA[0]!.header.hash, nodeName: 'node3' }, + ]); }); test.serial('[e2e] handles reversal of single-block re-org', async (t) => { @@ -806,10 +2329,13 @@ test.serial('[e2e] handles reversal of single-block re-org', async (t) => { /node3: re-organization detected beginning at height: 3001. The following stale blocks were removed:/u ); await waitForStdout(/Saved new block – height:\s+3006[^\n]+nodes: node1/u); - await waitForStdout( - /Saved new block – height:\s+3006[^\n]+nodes: node2, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipB[tipEnd - 1]!.header.hash, [ + 'node2', + 'node3', + ]), + ['node2', 'node3'] ); - t.pass(); }); test.serial('[e2e] handles re-org of 6 blocks', async (t) => { @@ -825,10 +2351,13 @@ test.serial('[e2e] handles re-org of 6 blocks', async (t) => { await waitForStdout( /node3: re-organization detected beginning at height: 3001. The following stale blocks were removed:/u ); - await waitForStdout( - /Saved new block – height:\s+3007[^\n]+nodes: node1, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipA[tipEnd - 1]!.header.hash, [ + 'node1', + 'node3', + ]), + ['node1', 'node3'] ); - t.pass(); }); test.serial('[e2e] handles reversal of 6 block re-org', async (t) => { @@ -844,10 +2373,13 @@ test.serial('[e2e] handles reversal of 6 block re-org', async (t) => { await waitForStdout( /node3: re-organization detected beginning at height: 3001. The following stale blocks were removed:/u ); - await waitForStdout( - /Saved new block – height:\s+3008[^\n]+nodes: node2, node3/u + t.deepEqual( + await waitForBlockAcceptedByNodes(tipB[tipEnd - 1]!.header.hash, [ + 'node2', + 'node3', + ]), + ['node2', 'node3'] ); - t.pass(); }); /** @@ -984,20 +2516,21 @@ test.serial( peers.node1.sendMessage(new peers.node1.messages.Transaction(tx2)); logger.debug(`node1: sent tx2: ${tx2!.hash}`); peers.node1.sendMessage(new peers.node1.messages.Transaction(tx3)); - logger.debug(`node1: sent tx3: ${tx2!.hash}`); - const delay = 100; - await sleep(delay); - const mempool1 = await client.query<{ encode: string }>( - /* sql */ `SELECT encode(hash, 'hex') FROM node_transaction JOIN transaction ON node_transaction.transaction_internal_id = transaction.internal_id ORDER BY hash ASC;` - ); - t.deepEqual(mempool1.rows, [ - { encode: tx1!.hash }, - { encode: tx3!.hash }, - { encode: tx2!.hash }, - { encode: chipnetCashTokensTxHash }, - { encode: halTxSpent }, - { encode: halTxHash }, - ]); + logger.debug(`node1: sent tx3: ${tx3!.hash}`); + const expectedMempool = [ + tx1!.hash, + tx3!.hash, + tx2!.hash, + chipnetCashTokensTxHash, + halTxSpent, + halTxHash, + ]; + t.deepEqual( + await waitForMempoolTransactionHashes( + (hashes) => hashes.join() === expectedMempool.join() + ), + expectedMempool + ); newBlocks('node1', [tipA[161]!]); newBlocks('node2', [tipB[161]!]); newBlocks('node3', [tipA[161]!]); @@ -1009,13 +2542,10 @@ test.serial( await waitForStdout( /Saved new block – height:\s+3162[^\n]+nodes: node1, node3/u ); - const mempool2 = await client.query<{ encode: string }>( - /* sql */ `SELECT encode(hash, 'hex') FROM node_transaction JOIN transaction ON node_transaction.transaction_internal_id = transaction.internal_id ORDER BY hash ASC;` - ); - t.deepEqual(mempool2.rows, [ - { encode: chipnetCashTokensTxHash }, - { encode: halTxSpent }, - { encode: halTxHash }, + t.deepEqual(await getMempoolTransactionHashes(), [ + chipnetCashTokensTxHash, + halTxSpent, + halTxHash, ]); t.pass(); } @@ -1029,6 +2559,51 @@ test.serial('[e2e] shuts down with SIGINT', async (t) => { t.pass(); }); +const historicalRepairTipIndex = 161; +const historicalRepairTransactionIndex = 1; +const historicalRepairBlock = tipA[historicalRepairTipIndex]!; +const historicalRepairBlockHash = historicalRepairBlock.header.hash; + +test.serial( + '[e2e] prepares incomplete historical block transaction before restart', + async (t) => { + const transactionHash = + historicalRepairBlock.transactions[historicalRepairTransactionIndex]! + .hash; + const selectedTransaction = ( + await client.query<{ hash: string }>( + /* sql */ ` + SELECT encode(transaction.hash, 'hex') AS hash + FROM block_transaction + INNER JOIN block + ON block.internal_id = block_transaction.block_internal_id + INNER JOIN transaction + ON transaction.internal_id = + block_transaction.transaction_internal_id + WHERE block.hash = $1 + AND block_transaction.transaction_index = $2; + `, + [hexToBin(historicalRepairBlockHash), historicalRepairTransactionIndex] + ) + ).rows; + t.deepEqual(selectedTransaction, [{ hash: transactionHash }]); + await client.query( + /* sql */ ` + DELETE FROM block_transaction + USING block + WHERE block.internal_id = block_transaction.block_internal_id + AND block.hash = $1 + AND block_transaction.transaction_index = $2; + `, + [hexToBin(historicalRepairBlockHash), historicalRepairTransactionIndex] + ); + t.deepEqual( + await getBlockTransactionCount(historicalRepairBlockHash), + historicalRepairBlock.transactions.length - 1 + ); + } +); + test.serial( '[e2e] restores sync-state from database on restart (after initial sync)', async (t) => { @@ -1078,6 +2653,218 @@ test.serial('[e2e] catches up a new node via headers', async (t) => { t.pass(); }); +test.serial( + '[e2e] self-heals incomplete historical block transactions on startup', + async (t) => { + t.timeout(oneMinute); + t.deepEqual( + await waitForBlockTransactionCount( + historicalRepairBlockHash, + historicalRepairBlock.transactions.length + ), + historicalRepairBlock.transactions.length + ); + } +); + +const getIncompleteBlockAuditMarkers = async () => + ( + await client.query<{ + auditedThroughBlockInternalId: string; + nodeInternalId: number; + nodeName: string; + }>(/* sql */ ` + SELECT node.name AS "nodeName", + node.internal_id AS "nodeInternalId", + incomplete_block_audit.audited_through_block_internal_id + AS "auditedThroughBlockInternalId" + FROM incomplete_block_audit + INNER JOIN node + ON node.internal_id = incomplete_block_audit.node_internal_id + ORDER BY node.name; + `) + ).rows; + +const waitForIncompleteBlockAuditMarker = async ( + nodeName: string, + remainingAttempts = blockRepairPollingAttempts +): Promise>> => { + const markers = await getIncompleteBlockAuditMarkers(); + if ( + remainingAttempts <= 0 || + markers.some((marker) => marker.nodeName === nodeName) + ) { + return markers; + } + await sleep(blockRepairPollingIntervalMs); + return waitForIncompleteBlockAuditMarker(nodeName, remainingAttempts - 1); +}; + +test.serial( + '[e2e] records incomplete block audit markers and skips audited blocks on restart', + async (t) => { + t.timeout(oneMinute); + const markers = await waitForIncompleteBlockAuditMarker('node4'); + const markersByNode = new Map( + markers.map((marker) => [marker.nodeName, marker]) + ); + ['node1', 'node2', 'node4'].forEach((nodeName) => { + t.true(markersByNode.has(nodeName), `missing marker for ${nodeName}`); + }); + const maxBlockInternalId = Number( + ( + await client.query<{ max: string }>( + /* sql */ `SELECT MAX(internal_id) AS max FROM block;` + ) + ).rows[0]!.max + ); + markers.forEach((marker) => { + const auditedThrough = Number(marker.auditedThroughBlockInternalId); + t.true(auditedThrough > 0 && auditedThrough <= maxBlockInternalId); + }); + /* + * The previous agent process completed a scan for node1 and node2, so this + * process loaded their markers (and skipped their audited blocks); node4 is + * new, so all of its blocks were audited (which repaired the incomplete + * historical block above). + */ + t.regex( + stdoutBuffer, + new RegExp( + `starting incomplete block repair scan;[^\\n]*${ + markersByNode.get('node1')!.nodeInternalId + }: \\d+[^\\n]*${markersByNode.get('node4')!.nodeInternalId}: none`, + 'u' + ) + ); + } +); + +test.serial( + '[e2e] getIncompleteBlocks skips blocks already audited for a node', + async (t) => { + const originalPostgresConnectionString = + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + postgresE2eConnectionStringTestDb; + /* + * An earlier test ends the shared `db.js` pool, so load a fresh module + * instance (with its own pool) for this test. + */ + const freshDbModuleUrl = `../db.js?incomplete-block-audit=${Date.now()}`; + const { getIncompleteBlocks, pool: freshDbPool } = (await import( + freshDbModuleUrl + )) as typeof DbModule; + const markersByNode = new Map( + (await getIncompleteBlockAuditMarkers()).map((marker) => [ + marker.nodeName, + marker, + ]) + ); + const auditedNode1 = markersByNode.get('node1')!; + const auditedNode4 = markersByNode.get('node4')!; + const removedRow = ( + await client.query<{ + blockInternalId: string; + transactionIndex: number; + transactionInternalId: string; + }>( + /* sql */ ` + DELETE FROM block_transaction + USING block + WHERE block.internal_id = block_transaction.block_internal_id + AND block.hash = $1 + AND block_transaction.transaction_index = $2 + RETURNING block_transaction.block_internal_id AS "blockInternalId", + block_transaction.transaction_internal_id + AS "transactionInternalId", + block_transaction.transaction_index AS "transactionIndex"; + `, + [hexToBin(historicalRepairBlockHash), historicalRepairTransactionIndex] + ) + ).rows[0]!; + // eslint-disable-next-line functional/no-try-statement + try { + t.true( + Number(removedRow.blockInternalId) <= + Number(auditedNode1.auditedThroughBlockInternalId) + ); + const height = historicalRepairTipIndex + splitHeight + 1; + const scan = async ( + nodeInternalIds: number[], + auditedThroughBlockInternalIds?: number[] + ) => + getIncompleteBlocks({ + auditedThroughBlockInternalIds, + excludedBlockHashes: [], + heightLowerBound: height, + heightUpperBound: height + 1, + limit: 10, + nodeInternalIds, + }); + t.deepEqual( + await scan( + [auditedNode1.nodeInternalId], + [Number(auditedNode1.auditedThroughBlockInternalId)] + ), + { incompleteBlocks: [], scannedBlockCount: 0 } + ); + const unaudited = await scan([auditedNode1.nodeInternalId]); + t.deepEqual(unaudited.scannedBlockCount, 1); + t.deepEqual( + unaudited.incompleteBlocks.map((block) => block.hash), + [historicalRepairBlockHash] + ); + const oneNodeUnaudited = await scan( + [auditedNode1.nodeInternalId, auditedNode4.nodeInternalId], + [Number(auditedNode1.auditedThroughBlockInternalId), 0] + ); + t.deepEqual( + oneNodeUnaudited.incompleteBlocks.map((block) => block.hash), + [historicalRepairBlockHash] + ); + /* + * All markers positive (the bounded per-node join, not the full-audit + * fallback): node4's marker is below the block, so the block is still + * returned although node1 has already audited it. + */ + const blockInternalId = Number(removedRow.blockInternalId); + t.true(blockInternalId > 1); + const oneNodeAuditedBelow = await scan( + [auditedNode1.nodeInternalId, auditedNode4.nodeInternalId], + [ + Number(auditedNode1.auditedThroughBlockInternalId), + blockInternalId - 1, + ] + ); + t.deepEqual( + oneNodeAuditedBelow.incompleteBlocks.map((block) => block.hash), + [historicalRepairBlockHash] + ); + } finally { + await client.query( + /* sql */ ` + INSERT INTO block_transaction + (block_internal_id, transaction_internal_id, transaction_index) + VALUES ($1, $2, $3); + `, + [ + removedRow.blockInternalId, + removedRow.transactionInternalId, + removedRow.transactionIndex, + ] + ); + await freshDbPool.end(); + if (originalPostgresConnectionString === undefined) { + delete process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING; + } else { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = + originalPostgresConnectionString; + } + } + } +); + test.serial( '[e2e] handles empty headers messages (fully-synced)', async (t) => { @@ -1089,8 +2876,54 @@ test.serial( } ); +test.serial( + '[e2e] saves block transactions if previously announced tx is seen but not yet saved', + async (t) => { + const tipStartIndex = 162; + const [, tx1] = tipA[tipStartIndex]!.transactions; + mempool[swapEndianness(tx1!.hash)] = false; + const node1RequestedTx = new Promise((res) => { + node1.once('peergetdata', (_, message) => { + res(message.inventory); + }); + }); + peers.node1.sendMessage( + peers.node1.messages.Inventory.forTransaction( + Buffer.from(tx1!.hash, 'hex') + ) + ); + await node1RequestedTx; + logger.debug( + `node1: announced tipA[${tipStartIndex}] transaction 1 without providing the transaction: ${ + tx1!.hash + }` + ); + newBlocks('node1', [tipA[tipStartIndex]!]); + newBlocks('node2', [tipB[tipStartIndex]!]); + newBlocks('node3', [tipA[tipStartIndex]!]); + await waitForStdout(/Saved new block – height:\s+3163[^\n]+nodes: node2/u); + await waitForStdout( + /Saved new block – height:\s+3163[^\n]+nodes: node1, node4/u + ); + const blockTransactionCount = ( + await client.query<{ count: string }>( + /* sql */ ` + SELECT COUNT(*) FROM block_transaction + INNER JOIN block ON block.internal_id = block_transaction.block_internal_id + WHERE block.hash = $1; + `, + [hexToBin(tipA[tipStartIndex]!.header.hash)] + ) + ).rows[0]!.count; + t.deepEqual( + blockTransactionCount, + tipA[tipStartIndex]!.transactions.length.toString() + ); + } +); + test.serial('[e2e] syncs remaining blocks one-by-one', async (t) => { - const tipStartIndex = 162; + const tipStartIndex = 163; slowFeedBlocks('node1', tipA.slice(tipStartIndex), 1); slowFeedBlocks('node2', tipB.slice(tipStartIndex), 1); /** @@ -1360,6 +3193,18 @@ const bytecodeFunction = test.macro<[string, string, string]>({ `[e2e] [postgres] ${functionName} – ${patternHex}: ${providedTitle ?? ''}`, }); +const bytecodeFunctionReturnsNull = test.macro<[string, string]>({ + exec: async (t, functionName, bytecodeHex) => { + const result = await client.query<{ isNull: boolean }>( + /* sql */ `SELECT ${functionName} ($1) IS NULL AS "isNull";`, + [hexToBin(bytecodeHex)] + ); + t.true(result.rows[0]!.isNull); + }, + title: (providedTitle, functionName, bytecodeHex) => + `[e2e] [postgres] ${functionName} – ${bytecodeHex}: ${providedTitle ?? ''}`, +}); + test( 'P2PKH', bytecodeFunction, @@ -1470,6 +3315,28 @@ test( '6a02094c5c' ); +test( + 'zero-length OP_PUSHDATA1', + bytecodeFunction, + 'parse_bytecode_pattern_with_pushdata_lengths', + '4c00', + '4c00' +); +test( + 'zero-length OP_PUSHDATA2', + bytecodeFunction, + 'parse_bytecode_pattern_with_pushdata_lengths', + '4d0000', + '4d0000' +); +test( + 'zero-length OP_PUSHDATA4', + bytecodeFunction, + 'parse_bytecode_pattern_with_pushdata_lengths', + '4e00000000', + '4e00000000' +); + test( 'OP_RETURN with OP_PUSHDATA2', bytecodeFunction, @@ -1521,10 +3388,9 @@ test( test( 'no redeem', - bytecodeFunction, + bytecodeFunctionReturnsNull, 'parse_bytecode_pattern_redeem', - `0002000051`, - '' + `0002000051` ); test( @@ -1584,6 +3450,31 @@ test( '004e515253' ); +test( + 'malformed OP_PUSHDATA1 redeem', + bytecodeFunctionReturnsNull, + 'parse_bytecode_pattern_redeem', + '4c' +); +test( + 'malformed OP_PUSHDATA2 redeem', + bytecodeFunctionReturnsNull, + 'parse_bytecode_pattern_redeem', + '4d11' +); +test( + 'malformed OP_PUSHDATA4 redeem', + bytecodeFunctionReturnsNull, + 'parse_bytecode_pattern_redeem', + '4e112233' +); +test( + 'oversized OP_PUSHDATA4 redeem', + bytecodeFunctionReturnsNull, + 'parse_bytecode_pattern_redeem', + '4effffffff' +); + test('[e2e] [postgres] encode_uint16le', async (t) => { const query = async (encoded: number) => ( @@ -1724,3 +3615,170 @@ test('[e2e] [postgres] encode_compact_uint', async (t) => { t.deepEqual(await query(BigInt('9223372036854775807')), 'ffffffffffffffff7f'); /* eslint-enable @typescript-eslint/no-magic-numbers */ }); + +/* cspell: disable */ +const searchFixtureTxHash = + 'f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0'; +const searchFixtureOutputs = { + /** P2PKH whose hash contains 0x5c (the LIKE escape character) */ + backslash: '76a9144444444444444444445c4444444444444444444488ac', + /** P2PKH that only a wildcard reading of 0x25 or 0x5f would match */ + other: '76a914997777777777777777777777777777777777777788ac', + /** 25-byte P2PKH */ + p2pkh: '76a914111111111111111111111111111111111111111188ac', + /** a longer script sharing the P2PKH's first 25 bytes */ + p2pkhPrefixed: '76a914111111111111111111111111111111111111111188acab', + /** 35-byte P2SH32 */ + p2sh32: + 'aa20222222222222222222222222222222222222222222222222222222222222222287', + /** shares the P2SH32's first 25 bytes, differs afterwards */ + p2sh32SamePrefix: + 'aa20222222222222222222222222222222222222222222333333333333333333333387', + /** P2PKH whose hash starts with 0x25 (LIKE "%") */ + percent: '76a914255555555555555555555555555555555555555588ac', + /** P2PKH whose hash starts with 0x5f (LIKE "_") */ + underscore: '76a9145f6666666666666666666666666666666666666688ac', +}; +/* cspell: enable */ + +/** + * Insert synthetic outputs in a database transaction that is always rolled + * back, run `query` restricted to the fixture transaction, and return the + * names of the matched fixture outputs. + */ +const searchFixtureMatches = async (query: string, parameter: unknown) => { + const names = Object.keys(searchFixtureOutputs); + await client.query('BEGIN;'); + const insertAndQuery = async () => { + await client.query( + /* sql */ `INSERT INTO transaction (hash, version, locktime, size_bytes, is_coinbase) VALUES ($1, 2, 0, 0, false);`, + [hexToBin(searchFixtureTxHash)] + ); + await Object.values(searchFixtureOutputs).reduce>( + async (chain, bytecode, index) => + chain.then(async () => + client.query( + /* sql */ `INSERT INTO output (transaction_hash, output_index, value_satoshis, locking_bytecode) VALUES ($1, $2, 1000, $3);`, + [hexToBin(searchFixtureTxHash), index, hexToBin(bytecode)] + ) + ), + Promise.resolve(undefined) + ); + const result = await client.query<{ outputIndex: string }>( + /* sql */ `SELECT output_index AS "outputIndex" FROM ${query} AS o WHERE o.transaction_hash = $2 ORDER BY output_index;`, + [parameter, hexToBin(searchFixtureTxHash)] + ); + return result.rows.map((row) => names[Number(row.outputIndex)]); + }; + const rollback = async () => client.query('ROLLBACK;'); + return insertAndQuery().then( + async (matches) => rollback().then(() => matches), + // eslint-disable-next-line functional/no-promise-reject -- roll back, then propagate the original failure + async (error: unknown) => rollback().then(async () => Promise.reject(error)) + ); +}; + +test.serial( + '[e2e] [sql] search_output: exact matches for locking bytecode of any length', + async (t) => { + const search = async (scripts: string[]) => + searchFixtureMatches('search_output($1::text[])', scripts); + t.deepEqual(await search([searchFixtureOutputs.p2sh32]), ['p2sh32']); + t.deepEqual(await search([searchFixtureOutputs.p2pkh]), ['p2pkh']); + t.deepEqual(await search([searchFixtureOutputs.p2pkhPrefixed]), [ + 'p2pkhPrefixed', + ]); + t.deepEqual( + await search([ + searchFixtureOutputs.p2sh32SamePrefix, + searchFixtureOutputs.p2pkh, + searchFixtureOutputs.p2pkh, + ]), + ['p2pkh', 'p2sh32SamePrefix'] + ); + t.deepEqual(await search([]), []); + } +); + +/* cspell: disable */ +test.serial( + '[e2e] [sql] search_output_prefix: treats every byte literally and accepts prefixes longer than 25 bytes', + async (t) => { + const search = async (prefix: string) => + searchFixtureMatches('search_output_prefix($1::text)', prefix); + t.deepEqual(await search(searchFixtureOutputs.backslash), ['backslash']); + t.deepEqual(await search('76a9144444444444444444445c'), ['backslash']); + t.deepEqual(await search('76a91425'), ['percent']); + t.deepEqual(await search('76a9145f'), ['underscore']); + t.deepEqual(await search(searchFixtureOutputs.p2sh32), ['p2sh32']); + t.deepEqual(await search(searchFixtureOutputs.p2pkh), [ + 'p2pkh', + 'p2pkhPrefixed', + ]); + t.deepEqual(await search('aa20'), ['p2sh32', 'p2sh32SamePrefix']); + } +); +/* cspell: enable */ + +test.serial( + '[e2e] [sql] search functions use the 25-byte locking bytecode prefix index', + async (t) => { + const prefixIndexes = Object.entries(indexDefinitions).filter( + ([, definition]) => definition.includes('locking_bytecode') + ); + t.deepEqual( + prefixIndexes.map(([name]) => name), + ['output_search_index'] + ); + prefixIndexes.forEach(([name, definition]) => { + t.true(definition.includes('substring(locking_bytecode, 0, 26)'), name); + }); + await client.query('BEGIN;'); + const explainPlans = async () => { + await prefixIndexes.reduce>( + async (chain, [name, definition]) => + chain.then(async () => + client.query( + definition.replace( + `CREATE INDEX ${name}`, + `CREATE INDEX test_${name}` + ) + ) + ), + Promise.resolve(undefined) + ); + // cspell: disable-next-line + await client.query('SET LOCAL enable_seqscan = off;'); + const plan = async (query: string) => + ( + await client.query<{ [column: string]: string }>( + `EXPLAIN (COSTS OFF) ${query}` + ) + ).rows + .map((row) => Object.values(row).join('')) + .join('\n'); + return [ + // cspell: disable-next-line + await plan(`SELECT * FROM search_output(ARRAY['76a91411'])`), + // cspell: disable-next-line + await plan(`SELECT * FROM search_output_prefix('76a914')`), + ]; + }; + const rollback = async () => client.query('ROLLBACK;'); + const plans = await explainPlans().then( + async (result) => rollback().then(() => result), + async (error: unknown) => + // eslint-disable-next-line functional/no-promise-reject -- roll back, then propagate the original failure + rollback().then(async () => Promise.reject(error)) + ); + /* + * Either copy: the two are identical, and once Postgres has auto-analyzed + * `output` the managed index has expression statistics the fresh copy + * lacks, which tips the planner's tie towards it. + */ + plans.forEach((plan) => { + t.true(/\b(?:test_)?output_search_index\b/u.test(plan), plan); + t.false(plan.includes('Seq Scan'), plan); + }); + } +); diff --git a/src/e2e/mempool-cleanup.spec.ts b/src/e2e/mempool-cleanup.spec.ts new file mode 100644 index 00000000..7a13ec3b --- /dev/null +++ b/src/e2e/mempool-cleanup.spec.ts @@ -0,0 +1,517 @@ +import { readFileSync } from 'fs'; + +import type { TestFn } from 'ava'; +import baseTest from 'ava'; + +import type * as database from '../db.js'; + +// cspell:words lpad +const test = baseTest as TestFn<{ db: typeof database }>; +const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; +const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; + +/* + * A single connection keeps every operation inside the same temporary schema, + * discarded after each test (not a rollback transaction: expiry runs its own + * transaction). No persistent tables are created or modified. + */ +test.before(async (t) => { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = `postgres://chaingraph:very_insecure_postgres_password@${host}:${port}/postgres`; + process.env.CHAINGRAPH_POSTGRES_MAX_CONNECTIONS = '1'; + t.context.db = await import('../db.js'); +}); + +test.beforeEach(async (t) => { + await t.context.db.pool.query(/* sql */ ` +SET search_path = pg_temp; +SET statement_timeout = '5s'; +CREATE TEMP TABLE node (internal_id integer PRIMARY KEY, name text); +CREATE TEMP TABLE transaction (internal_id bigint PRIMARY KEY, hash bytea UNIQUE); +CREATE TEMP TABLE output (transaction_hash bytea, output_index integer, + PRIMARY KEY (transaction_hash, output_index)); +CREATE TEMP TABLE input (transaction_internal_id bigint, input_index integer, + outpoint_transaction_hash bytea, outpoint_index integer, + PRIMARY KEY (transaction_internal_id, input_index)); +CREATE INDEX input_outpoint ON input (outpoint_transaction_hash, outpoint_index); +CREATE TEMP TABLE node_transaction (node_internal_id integer, + transaction_internal_id bigint, validated_at timestamp DEFAULT '2026-01-01', + PRIMARY KEY (node_internal_id, transaction_internal_id)); +CREATE TEMP TABLE node_transaction_history (node_internal_id integer, + transaction_internal_id bigint, validated_at timestamp, replaced_at timestamp, + PRIMARY KEY (node_internal_id, transaction_internal_id, validated_at)); +CREATE TEMP TABLE block_transaction (block_internal_id bigint, + transaction_internal_id bigint, + PRIMARY KEY (transaction_internal_id, block_internal_id)); +CREATE TEMP TABLE node_block (node_internal_id integer, + block_internal_id bigint, accepted_at timestamp, + PRIMARY KEY (node_internal_id, block_internal_id)); +INSERT INTO node VALUES (1, 'alpha'), (2, 'beta'); +INSERT INTO transaction + SELECT id, decode(repeat(lpad(to_hex(id), 2, '0'), 32), 'hex') + FROM generate_series(1, 20) id; +INSERT INTO output SELECT hash, 0 FROM transaction; +`); + const migration = readFileSync( + new URL( + '../../images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql', + import.meta.url + ), + 'utf8' + ); + const cascadeFunction = migration + .split('CREATE TRIGGER')[0]! + .replace( + 'FUNCTION trigger_node_transaction_history_insert()', + 'FUNCTION pg_temp.trigger_node_transaction_history_insert()' + ); + await t.context.db.pool.query(`${cascadeFunction} +CREATE TRIGGER cascade_history AFTER INSERT ON node_transaction_history + REFERENCING NEW TABLE AS new_table FOR EACH STATEMENT + EXECUTE FUNCTION pg_temp.trigger_node_transaction_history_insert(); +`); +}); + +test.afterEach.always(async (t) => { + await t.context.db.pool.query('DISCARD TEMP;'); +}); + +test.after.always(async (t) => { + await t.context.db.pool.end(); +}); + +const repeatedHashBytes = 32; +const hash = (byte: string) => byte.repeat(repeatedHashBytes); + +const membershipQuery = /* sql */ ` +SELECT node_internal_id AS node, transaction_internal_id::integer AS tx + FROM node_transaction ORDER BY node, tx; +`; +const historyQuery = /* sql */ ` +SELECT node_internal_id AS node, transaction_internal_id::integer AS tx, + replaced_at::text AS replaced + FROM node_transaction_history ORDER BY node, tx; +`; + +test.serial( + '[e2e] cleanup confirms a parent without invalidating descendants or another node', + async (t) => { + const { pool, archiveMempoolTransactionsAcceptedByBlocks: archive } = + t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + SELECT node, tx FROM generate_series(1, 2) node, generate_series(1, 3) tx; +-- Duplicate accepted inclusions must still produce only one history row. +INSERT INTO block_transaction VALUES (1, 1), (2, 1); +INSERT INTO node_block VALUES (1, 1, '2026-01-02'), (1, 2, '2026-01-03'); +`); + t.deepEqual(await archive(), [ + { hash: hash('01'), nodeName: 'alpha', replacedAt: null }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 1, tx: 2 }, + { node: 1, tx: 3 }, + { node: 2, tx: 1 }, + { node: 2, tx: 2 }, + { node: 2, tx: 3 }, + ]); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: null, tx: 1 }, + ]); + t.deepEqual(await archive(), []); + } +); + +test.serial( + '[e2e] cleanup invalidates conflicts and descendants only for the accepting node', + async (t) => { + const { pool, archiveMempoolTransactionsAcceptedByBlocks: archive } = + t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 1, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + SELECT node, tx FROM generate_series(1, 2) node, generate_series(1, 3) tx; +INSERT INTO block_transaction VALUES (1, 4), (2, 4); +INSERT INTO node_block VALUES (1, 1, '2026-01-05'), (1, 2, '2026-01-02'); +`); + t.deepEqual(await archive(), [ + { + hash: hash('01'), + nodeName: 'alpha', + replacedAt: new Date('2026-01-02T00:00:00.000Z'), + }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 2, tx: 1 }, + { node: 2, tx: 2 }, + { node: 2, tx: 3 }, + ]); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: '2026-01-02 00:00:00', tx: 1 }, + { node: 1, replaced: '2026-01-02 00:00:00', tx: 2 }, + { node: 1, replaced: '2026-01-02 00:00:00', tx: 3 }, + ]); + t.deepEqual(await archive(), []); + } +); + +test.serial( + '[e2e] cleanup gives confirmation precedence in mixed confirmation and invalidation batches', + async (t) => { + const { pool, archiveMempoolTransactionsAcceptedByBlocks: archive } = + t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 1, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 8, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 6, 0, hash, 0 FROM transaction WHERE internal_id = 5 + UNION ALL SELECT 7, 0, hash, 0 FROM transaction WHERE internal_id = 6; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 1), (1, 2), (1, 5), (1, 6), (1, 7); +-- Deliberately inconsistent historical acceptance tests NULL precedence. +INSERT INTO block_transaction VALUES (1, 1), (2, 4), (3, 8); +INSERT INTO node_block VALUES + (1, 1, '2026-01-02'), (1, 2, '2026-01-03'), (1, 3, '2026-01-04'); +`); + t.deepEqual(await archive(), [ + { hash: hash('01'), nodeName: 'alpha', replacedAt: null }, + { + hash: hash('05'), + nodeName: 'alpha', + replacedAt: new Date('2026-01-04T00:00:00.000Z'), + }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, [{ node: 1, tx: 2 }]); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: null, tx: 1 }, + { node: 1, replaced: '2026-01-04 00:00:00', tx: 5 }, + { node: 1, replaced: '2026-01-04 00:00:00', tx: 6 }, + { node: 1, replaced: '2026-01-04 00:00:00', tx: 7 }, + ]); + } +); + +test.serial( + '[e2e] cleanup ignores other-node confirmations, self, coinbase and unconfirmed conflicts', + async (t) => { + const { pool, archiveMempoolTransactionsAcceptedByBlocks: archive } = + t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 1, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 2, 0, decode(repeat('00', 32), 'hex'), 0 + UNION ALL SELECT 3, 0, decode(repeat('00', 32), 'hex'), 0 + UNION ALL SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 6, 0, hash, 0 FROM transaction WHERE internal_id = 12 + UNION ALL SELECT 7, 0, hash, 0 FROM transaction WHERE internal_id = 12 + UNION ALL SELECT 8, 0, hash, 1 FROM transaction WHERE internal_id = 13 + UNION ALL SELECT 9, 0, hash, 0 FROM transaction WHERE internal_id = 13; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 1), (1, 2), (1, 4), (1, 6), (1, 8); +INSERT INTO block_transaction VALUES (1, 1), (2, 3), (3, 5), (4, 9); +INSERT INTO node_block VALUES + (2, 1, '2026-01-02'), (1, 2, '2026-01-02'), + (2, 3, '2026-01-02'), (1, 4, '2026-01-02'); +`); + t.deepEqual(await archive(), []); + t.deepEqual((await pool.query(historyQuery)).rows, []); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 1, tx: 1 }, + { node: 1, tx: 2 }, + { node: 1, tx: 4 }, + { node: 1, tx: 6 }, + { node: 1, tx: 8 }, + ]); + } +); + +test.serial('[e2e] cleanup accepts an empty mempool', async (t) => { + t.deepEqual( + await t.context.db.archiveMempoolTransactionsAcceptedByBlocks(), + [] + ); +}); + +/* + * Regression: the agent's mempool scan runs archiveMempoolTransactionsAcceptedByBlocks + * and then getMempoolTransactionsExpiringBefore as two separate statements, and + * expires the rows returned by the second via archiveMempoolTransaction. A row + * inserted between the two (e.g. an announcement of an already-confirmed + * transaction with an old validated_at) used to be archived as expired, with + * replaced_at = expiry, although the node had accepted it in a block. Expiry + * must leave such rows for the accepted-block archive. + */ +test.serial( + '[e2e] expiry never archives a transaction accepted or replaced by a block for that node', + async (t) => { + const { + pool, + archiveMempoolTransaction, + archiveMempoolTransactionsAcceptedByBlocks: archive, + getMempoolTransactionsExpiringBefore, + } = t.context.db; + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 10; +-- tx 1 is confirmed and tx 5 (conflicting with tx 4) is mined, for alpha only. +INSERT INTO block_transaction VALUES (1, 1), (2, 5); +INSERT INTO node_block VALUES (1, 1, '2026-01-02'), (1, 2, '2026-01-03'); +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 3); +`); + // Scan step 1: the accepted-block archive finds nothing to do yet. + t.deepEqual(await archive(), []); + // A concurrent insert lands between the two scan steps. + await pool.query(/* sql */ ` +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 1), (1, 4), (2, 1), (2, 4); +`); + // Scan step 2: every row is already past expiry, so all get timers. + const expiring = await getMempoolTransactionsExpiringBefore({ + expirationMs: 3_600_000, + expiresBefore: new Date('2026-01-10T00:00:00.000Z'), + }); + t.deepEqual( + expiring.map((row) => ({ + node: row.nodeName, + tx: row.transactionInternalId, + })), + [ + { node: 'alpha', tx: 1 }, + { node: 'alpha', tx: 3 }, + { node: 'alpha', tx: 4 }, + { node: 'beta', tx: 1 }, + { node: 'beta', tx: 4 }, + ] + ); + // Timers fire: only rows not resolved by a block accepted by that node expire. + const archivedCounts = await expiring.reduce>( + async (previous, row) => [ + ...(await previous), + await archiveMempoolTransaction({ + nodeInternalId: row.nodeInternalId, + replacedAt: row.expiresAt, + transactionInternalId: row.transactionInternalId, + }), + ], + Promise.resolve([]) + ); + t.deepEqual(archivedCounts, [0, 1, 0, 1, 1]); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 1, tx: 1 }, + { node: 1, tx: 4 }, + ]); + // The next scan archives them with the block-derived replaced_at. + t.deepEqual(await archive(), [ + { hash: hash('01'), nodeName: 'alpha', replacedAt: null }, + { + hash: hash('04'), + nodeName: 'alpha', + replacedAt: new Date('2026-01-03T00:00:00.000Z'), + }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, []); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: null, tx: 1 }, + { node: 1, replaced: '2026-01-01 01:00:00', tx: 3 }, + { node: 1, replaced: '2026-01-03 00:00:00', tx: 4 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 4 }, + ]); + } +); + +/* + * Regression: a batch holding an expired parent and an expired descendant + * deleted both directly, so the descendant (and, via the history trigger, + * everything below it) recorded its own, later expiry instead of the + * parent's earlier one. + */ +test.serial( + '[e2e] a batched expiry records the earliest ancestor expiry for every descendant', + async (t) => { + const { pool, archiveMempoolTransactions } = t.context.db; + // chain 1 <- 2 <- 3 <- 4; only 1 and 3 are in the batch + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2 + UNION ALL SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 3; +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + SELECT 1, tx FROM generate_series(1, 4) tx; +`); + const archived = await archiveMempoolTransactions([ + { + nodeInternalId: 1, + replacedAt: new Date('2026-01-01T02:00:00.000Z'), + transactionInternalId: 3, + }, + { + nodeInternalId: 1, + replacedAt: new Date('2026-01-01T01:00:00.000Z'), + transactionInternalId: 1, + }, + ]); + t.deepEqual( + archived.map((expiry) => ({ tx: expiry.transactionInternalId })), + [{ tx: 1 }, { tx: 3 }] + ); + t.deepEqual((await pool.query(membershipQuery)).rows, []); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 1, replaced: '2026-01-01 01:00:00', tx: 2 }, + { node: 1, replaced: '2026-01-01 01:00:00', tx: 3 }, + { node: 1, replaced: '2026-01-01 01:00:00', tx: 4 }, + ]); + } +); + +/* + * Regression: the block guard only covered the expired rows themselves. The + * history trigger then archived a descendant also replaced by a block accepted + * by the node (and the descendant's own descendants) with the parent's expiry, + * so the accepted-block archive could no longer record the block's accepted_at. + */ +test.serial( + '[e2e] expiry archives a descendant replaced by an accepted block with the block time', + async (t) => { + const { pool, archiveMempoolTransactions } = t.context.db; + // 2 spends 1 and 10; 5 (mined for alpha only) also spends 10; 3 spends 2 + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 2, 1, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO block_transaction VALUES (1, 5); +INSERT INTO node_block VALUES (1, 1, '2026-01-03'); +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + SELECT node, tx FROM generate_series(1, 2) node, generate_series(1, 3) tx; +`); + const replacedAt = new Date('2026-01-01T01:00:00.000Z'); + const archived = await archiveMempoolTransactions([ + { nodeInternalId: 1, replacedAt, transactionInternalId: 1 }, + { nodeInternalId: 2, replacedAt, transactionInternalId: 1 }, + ]); + t.deepEqual( + archived.map((expiry) => ({ node: expiry.nodeInternalId })), + [{ node: 1 }, { node: 2 }] + ); + t.deepEqual((await pool.query(membershipQuery)).rows, []); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 1, replaced: '2026-01-03 00:00:00', tx: 2 }, + { node: 1, replaced: '2026-01-03 00:00:00', tx: 3 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 2 }, + { node: 2, replaced: '2026-01-01 01:00:00', tx: 3 }, + ]); + } +); + +/* + * Regression: blocks accepted before Chaingraph began monitoring have a NULL + * accepted_at, which MIN(accepted_at) could not tell apart from no conflicting + * block at all. Expiry then archived a root replaced by such a block with its + * expiry, and gave such a descendant its ancestor's expiry, instead of leaving + * the root for the accepted-block archive and giving the descendant the + * accepted-block archive's NULL (which, as there, ends the cascade). + */ +test.serial( + '[e2e] expiry treats a conflict in a block with a NULL accepted_at as replaced', + async (t) => { + const { + pool, + archiveMempoolTransactions, + archiveMempoolTransactionsAcceptedByBlocks: archive, + } = t.context.db; + // 4 and 5 (mined) spend 10; chain 1 <- 2 <- 3, 2 and 6 (mined) spend 11 + await pool.query(/* sql */ ` +INSERT INTO input + SELECT 4, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 5, 0, hash, 0 FROM transaction WHERE internal_id = 10 + UNION ALL SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1 + UNION ALL SELECT 2, 1, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 6, 0, hash, 0 FROM transaction WHERE internal_id = 11 + UNION ALL SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO block_transaction VALUES (1, 5), (2, 6); +INSERT INTO node_block VALUES (1, 1, NULL), (1, 2, NULL); +INSERT INTO node_transaction (node_internal_id, transaction_internal_id) + VALUES (1, 1), (1, 2), (1, 3), (1, 4); +`); + const replacedAt = new Date('2026-01-01T01:00:00.000Z'); + const archived = await archiveMempoolTransactions([ + { nodeInternalId: 1, replacedAt, transactionInternalId: 1 }, + { nodeInternalId: 1, replacedAt, transactionInternalId: 4 }, + ]); + t.deepEqual( + archived.map((expiry) => ({ tx: expiry.transactionInternalId })), + [{ tx: 1 }] + ); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 1, replaced: '2026-01-01 01:00:00', tx: 1 }, + { node: 1, replaced: null, tx: 2 }, + ]); + t.deepEqual(await archive(), [ + { hash: hash('04'), nodeName: 'alpha', replacedAt: null }, + ]); + t.deepEqual((await pool.query(membershipQuery)).rows, [{ node: 1, tx: 3 }]); + } +); + +/* + * Regression: an expiry scheduled for one validation also archived a later + * validation of the same transaction (e.g. one restored by a mempool resync + * after the first was confirmed and a re-org removed the block), and through the + * history trigger that validation's descendants. + */ +test.serial( + '[e2e] expiry leaves a transaction validated again after it was scheduled', + async (t) => { + const { pool, archiveMempoolTransactions } = t.context.db; + // 2 spends 1; node 1 re-validated 1 after the expiry was scheduled + await pool.query(/* sql */ ` +INSERT INTO input SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1; +INSERT INTO node_transaction VALUES + (1, 1, '2026-01-02 00:00:00.123456'), (1, 2, '2026-01-02'), + (2, 1, '2026-01-01 00:00:00.123456'); +`); + const replacedAt = new Date('2026-01-01T01:00:00.000Z'); + const archived = await archiveMempoolTransactions([ + { + nodeInternalId: 1, + replacedAt, + transactionInternalId: 1, + validatedAt: new Date('2026-01-01T00:00:00.123Z'), + }, + { + nodeInternalId: 2, + replacedAt, + transactionInternalId: 1, + validatedAt: new Date('2026-01-01T00:00:00.123Z'), + }, + ]); + t.deepEqual( + archived.map((expiry) => ({ node: expiry.nodeInternalId })), + [{ node: 2 }] + ); + t.deepEqual((await pool.query(membershipQuery)).rows, [ + { node: 1, tx: 1 }, + { node: 1, tx: 2 }, + ]); + t.deepEqual((await pool.query(historyQuery)).rows, [ + { node: 2, replaced: '2026-01-01 01:00:00', tx: 1 }, + ]); + } +); diff --git a/src/e2e/mempool-expiry-concurrency.spec.ts b/src/e2e/mempool-expiry-concurrency.spec.ts new file mode 100644 index 00000000..ce4695ff --- /dev/null +++ b/src/e2e/mempool-expiry-concurrency.spec.ts @@ -0,0 +1,543 @@ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import { readFileSync } from 'fs'; + +import type { ExecutionContext, TestFn } from 'ava'; +import baseTest from 'ava'; + +import type * as database from '../db.js'; + +// cspell:words autovacuum lpad xact +const test = baseTest as TestFn<{ db: typeof database; schema: string }>; +const host = process.env.CHAINGRAPH_E2E_POSTGRES_HOST ?? 'localhost'; +const port = process.env.CHAINGRAPH_E2E_POSTGRES_PORT ?? '5432'; +const schema = `expiry_concurrency_${process.pid}`; +const poolSize = 16; + +/* + * Real (committed) tables in a scratch schema, so many pool connections see the + * same rows and take real row locks: the stale-mempool expiry of a restarted + * agent fires one timer per expired transaction at once. + */ +test.before(async (t) => { + process.env.CHAINGRAPH_POSTGRES_CONNECTION_STRING = `postgres://chaingraph:very_insecure_postgres_password@${host}:${port}/postgres?options=${encodeURIComponent( + `-c search_path=${schema} -c deadlock_timeout=50ms` + )}`; + process.env.CHAINGRAPH_POSTGRES_MAX_CONNECTIONS = `${poolSize}`; + t.context.db = await import('../db.js'); + t.context.schema = schema; + const migration = readFileSync( + new URL( + '../../images/hasura/hasura-data/migrations/default/1778151011521_cascade_invalidate_mempool_descendants/up.sql', + import.meta.url + ), + 'utf8' + ); + const cascadeFunction = migration.split('CREATE TRIGGER')[0]!; + await t.context.db.pool.query(/* sql */ ` +DROP SCHEMA IF EXISTS ${schema} CASCADE; +CREATE SCHEMA ${schema}; +SET search_path = ${schema}; +CREATE TABLE node (internal_id integer PRIMARY KEY, name text); +CREATE TABLE transaction (internal_id bigint PRIMARY KEY, hash bytea UNIQUE); +CREATE TABLE output (transaction_hash bytea, output_index integer, + PRIMARY KEY (transaction_hash, output_index)); +CREATE TABLE input (transaction_internal_id bigint, input_index integer, + outpoint_transaction_hash bytea, outpoint_index integer, + PRIMARY KEY (transaction_internal_id, input_index)); +CREATE INDEX input_outpoint ON input (outpoint_transaction_hash, outpoint_index); +CREATE TABLE node_transaction (node_internal_id integer, + transaction_internal_id bigint, validated_at timestamp NOT NULL, + PRIMARY KEY (node_internal_id, transaction_internal_id)); +CREATE TABLE node_transaction_history (node_internal_id integer, + transaction_internal_id bigint, validated_at timestamp, replaced_at timestamp, + PRIMARY KEY (node_internal_id, transaction_internal_id, validated_at)); +CREATE TABLE block_transaction (block_internal_id bigint, + transaction_internal_id bigint, + PRIMARY KEY (transaction_internal_id, block_internal_id)); +CREATE TABLE node_block (node_internal_id integer, + block_internal_id bigint, accepted_at timestamp, + PRIMARY KEY (node_internal_id, block_internal_id)); +${cascadeFunction} +CREATE TRIGGER cascade_history AFTER INSERT ON node_transaction_history + REFERENCING NEW TABLE AS new_table FOR EACH STATEMENT + EXECUTE FUNCTION trigger_node_transaction_history_insert(); +`); +}); + +test.after.always(async (t) => { + await t.context.db.pool.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE;`); + await t.context.db.pool.end(); +}); + +/** The expiry queue's batch size (`expiryBatchSize` in db.ts). */ +const expiryQueueBatchSize = 500; +interface MempoolShape { + chains: number; + depth: number; +} +/* + * Big enough for several expiry queue batches and many concurrent single-row + * expiries of related rows; small enough that the single-row test, which + * serializes one transaction per row, stays fast on a slow CI runner. + */ +const mempoolShape: MempoolShape = { chains: 20, depth: 15 }; +const mempoolRowCount = ({ chains, depth }: MempoolShape) => chains * depth * 2; + +/** + * `chains` chains of `depth` mempool transactions (each spends output 0 of the + * previous one), every transaction also fanning out to the next chain's + * transaction at the same depth, all validated at the same moment for two + * nodes: a restarted agent's stale mempool, all expiring at once. + */ +const createStaleMempool = async ( + db: typeof database, + validatedAt = `'2026-01-01'::timestamp`, + { chains, depth } = mempoolShape +) => { + await db.pool.query(/* sql */ ` +TRUNCATE node, transaction, output, input, node_transaction, + node_transaction_history, block_transaction, node_block; +-- production node ids are integers; node registration consumes sequence +-- values even for known nodes, so ids above the smallint range are routine +INSERT INTO node VALUES (40001, 'alpha'), (40002, 'beta'); +INSERT INTO transaction + SELECT id, sha256(int8send(id)) FROM generate_series(1, ${chains * depth}) id; +INSERT INTO output SELECT hash, o FROM transaction, generate_series(0, 1) o; +-- tx (c, d) has internal id c * depth + d + 1 +INSERT INTO input + SELECT c * ${depth} + d + 1, 0, parent.hash, 0 + FROM generate_series(0, ${chains - 1}) c CROSS JOIN generate_series(1, ${ + depth - 1 + }) d + JOIN transaction parent ON parent.internal_id = c * ${depth} + d + UNION ALL + SELECT c * ${depth} + d + 1, 1, parent.hash, 1 + FROM generate_series(1, ${chains - 1}) c CROSS JOIN generate_series(0, ${ + depth - 1 + }) d + JOIN transaction parent ON parent.internal_id = (c - 1) * ${depth} + d + 1; +INSERT INTO node_transaction + SELECT node.internal_id, transaction.internal_id, ${validatedAt} + FROM node CROSS JOIN transaction; +`); +}; + +const countRows = async (db: typeof database) => { + const result = await db.pool.query<{ + current: string; + history: string; + }>(/* sql */ ` +SELECT (SELECT count(*) FROM node_transaction)::text AS current, + (SELECT count(*) FROM node_transaction_history)::text AS history;`); + return { + current: Number(result.rows[0]!.current), + history: Number(result.rows[0]!.history), + }; +}; + +const shuffled = (items: T[], seed: number) => { + const copy = [...items]; + // eslint-disable-next-line functional/no-let + let state = seed; + // eslint-disable-next-line functional/no-loop-statement, functional/no-let + for (let i = copy.length - 1; i > 0; i -= 1) { + // eslint-disable-next-line no-bitwise + state = (state * 1103515245 + 12345) & 0x7fffffff; + const j = state % (i + 1); + [copy[i], copy[j]] = [copy[j]!, copy[i]!]; + } + return copy; +}; + +const expectAllArchived = async ( + t: ExecutionContext<{ db: typeof database; schema: string }>, + errors: unknown[], + replaced = '2026-01-01 01:00:00' +) => { + t.deepEqual( + errors.map((err) => (err as { code?: string }).code ?? err), + [], + 'no expiry fails (40P01 = deadlock)' + ); + t.deepEqual(await countRows(t.context.db), { + current: 0, + history: mempoolRowCount(mempoolShape), + }); + const replacedAt = await t.context.db.pool.query<{ replaced: string }>( + /* sql */ `SELECT DISTINCT replaced_at::text AS replaced FROM node_transaction_history;` + ); + t.deepEqual(replacedAt.rows, [{ replaced }]); +}; + +const staleExpiries = async ( + db: typeof database, + validatedAt?: string, + shape = mempoolShape +) => { + await createStaleMempool(db, validatedAt, shape); + const expiring = await db.getMempoolTransactionsExpiringBefore({ + expirationMs: 3_600_000, + expiresBefore: new Date('2026-01-10T00:00:00.000Z'), + }); + return shuffled(expiring, 7).map((row) => ({ + nodeInternalId: row.nodeInternalId, + replacedAt: row.expiresAt, + transactionInternalId: row.transactionInternalId, + })); +}; + +/* + * Regression (2026-10-13 backfill lab): a restarted agent's 6-day-stale mempool + * expired all at once, one archive statement per timer on every pool + * connection. Archiving a transaction cascades to its descendants (history + * trigger), so a parent's and a descendant's expiry locked the same rows in + * opposite orders: >1,300 deadlocks and no net progress. Before the fix this + * test failed with dozens of 40P01 errors (or timed out). + */ +test.serial( + '[e2e] concurrent single-row expiries of a stale mempool archive every row without deadlocks', + async (t) => { + const { db } = t.context; + const expiries = await staleExpiries(db); + t.is(expiries.length, mempoolRowCount(mempoolShape)); + const errors: unknown[] = []; + await Promise.all( + expiries.map(async (expiry) => + db.archiveMempoolTransaction(expiry).catch((err: unknown) => { + errors.push(err); + }) + ) + ); + await expectAllArchived(t, errors); + } +); + +test.serial( + '[e2e] concurrent expiry batches serialize and archive every row', + async (t) => { + const { db } = t.context; + const expiries = await staleExpiries(db); + const batchCount = poolSize; + const errors: unknown[] = []; + await Promise.all( + Array.from({ length: batchCount }, async (_, batch) => + db + .archiveMempoolTransactions( + expiries.filter((_expiry, index) => index % batchCount === batch) + ) + .catch((err: unknown) => { + errors.push(err); + }) + ) + ); + await expectAllArchived(t, errors); + } +); + +test.serial( + '[e2e] the expiry queue archives a stale mempool firing at once in a few ordered batches', + async (t) => { + const { db } = t.context; + const expiries = await staleExpiries(db); + const errors: unknown[] = []; + const results = await Promise.all( + expiries.map(async (expiry) => + db.expireMempoolTransaction(expiry).catch((err: unknown) => { + errors.push(err); + return undefined; + }) + ) + ); + await expectAllArchived(t, errors); + const archivedDirectly = results.filter((archived) => archived === true); + t.true(archivedDirectly.length > 0); + t.is( + archivedDirectly.length + + results.filter((archived) => archived === false).length, + expiries.length + ); + // a duplicate request for an already-archived row resolves false + t.false(await db.expireMempoolTransaction(expiries[0]!)); + } +); + +/* + * Every transaction descends from transaction 1, the first validated, so every + * row must record its expiry, also when a descendant's expiry is queued before + * its ancestors' and the queue needs several batches. + */ +test.serial( + '[e2e] the expiry queue archives every descendant with its earliest ancestor expiry', + async (t) => { + const { db } = t.context; + const expiries = await staleExpiries( + db, + `'2026-01-01'::timestamp + transaction.internal_id * interval '1 second'` + ); + const errors: unknown[] = []; + await Promise.all( + expiries.map(async (expiry) => + db.expireMempoolTransaction(expiry).catch((err: unknown) => { + errors.push(err); + }) + ) + ); + await expectAllArchived(t, errors, '2026-01-01 01:00:01'); + } +); + +/* eslint-disable @typescript-eslint/naming-convention */ +interface PlanNode { + 'Actual Loops': number; + 'Actual Rows': number; + 'Node Type': string; + Plans?: PlanNode[]; + 'Relation Name'?: string; + 'Rows Removed by Filter'?: number; + 'Rows Removed by Index Recheck'?: number; +} +/* eslint-enable @typescript-eslint/naming-convention */ +const rowsReadPerLoop = (node: PlanNode) => + node['Actual Rows'] + + (node['Rows Removed by Filter'] ?? 0) + + (node['Rows Removed by Index Recheck'] ?? 0); +/** + * Rows of `relation` read by the scans of an EXPLAIN ANALYZE plan: those + * emitted plus those discarded by the scan's filter or index recheck (all + * reported per loop). + */ +const rowsScanned = (node: PlanNode, relation: string): number => + (node['Relation Name'] === relation && node['Node Type'].endsWith('Scan') + ? rowsReadPerLoop(node) * node['Actual Loops'] + : 0) + + (node.Plans ?? []).reduce( + (total, child) => total + rowsScanned(child, relation), + 0 + ); + +/* + * Regression: with node_transaction statistics taken when the mempool was + * nearly empty (as autovacuum may analyze it just after a block), Postgres + * joined each walked descendant to node_transaction by node alone, reading the + * node's whole mempool per walked row: archiving a stale mempool took time + * quadratic in its size (here ~485,000 node_transaction rows read rather than + * ~20,000; tens of seconds per batch on CI). + */ +test.serial( + '[e2e] archiving a stale mempool reads each node_transaction row a few times, also with outdated statistics', + async (t) => { + const { db } = t.context; + // larger, so the quadratic plan reads far more rows than the bound below + const largeMempool = { chains: 60, depth: 40 }; + const mempoolRows = mempoolRowCount(largeMempool); + // statistics of a nearly empty mempool, then the stale mempool + await createStaleMempool(db, undefined, largeMempool); + await db.pool.query(/* sql */ ` +DELETE FROM node_transaction WHERE transaction_internal_id > 3; +ANALYZE node_transaction;`); + // one expiry queue batch, including transaction 1: the ancestor of all + const expiries = (await staleExpiries(db, undefined, largeMempool)) + .sort((a, b) => a.transactionInternalId - b.transactionInternalId) + .slice(0, expiryQueueBatchSize); + const client = await db.pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + await client.query('BEGIN;'); + const explained = await client.query<{ + // eslint-disable-next-line @typescript-eslint/naming-convention + 'QUERY PLAN': [{ Plan: PlanNode }]; + }>( + `EXPLAIN (ANALYZE, FORMAT JSON) ${db.mempoolExpiryArchivalStatement}`, + [ + expiries.map((expiry) => expiry.nodeInternalId), + expiries.map((expiry) => expiry.transactionInternalId), + expiries.map((expiry) => expiry.replacedAt.toISOString()), + expiries.map(() => null), + ] + ); + const scanned = rowsScanned( + explained.rows[0]!['QUERY PLAN'][0].Plan, + 'node_transaction' + ); + t.log(`node_transaction rows scanned: ${scanned}`); + t.true(scanned <= mempoolRows * 10, `scanned ${scanned} rows`); + const archived = await client.query<{ count: string }>( + /* sql */ `SELECT count(*)::text FROM node_transaction_history;` + ); + t.is(Number(archived.rows[0]!.count), mempoolRows); + } finally { + await client.query('ROLLBACK;'); + client.release(); + } + } +); + +/* + * Expire transaction 1 of a two-transaction mempool for alpha (2 spends 1, and + * `prepare` runs first) while another connection's transaction, `blocker`, + * holds a lock the expiry waits for. Once the expiry waits, `concurrently` is + * committed on a third connection, then `blocker`. + */ +const expireWhileBlocked = async ( + db: typeof database, + { + blocker, + concurrently = '', + prepare = '', + }: { blocker: string; concurrently?: string; prepare?: string } +) => { + await db.pool.query(/* sql */ ` +TRUNCATE node, transaction, output, input, node_transaction, + node_transaction_history, block_transaction, node_block; +INSERT INTO node VALUES (40001, 'alpha'); +INSERT INTO transaction + SELECT id, sha256(int8send(id)) FROM generate_series(1, 2) id; +INSERT INTO output SELECT hash, 0 FROM transaction; +INSERT INTO input SELECT 2, 0, hash, 0 FROM transaction WHERE internal_id = 1; +INSERT INTO node_transaction + SELECT 40001, internal_id, '2026-01-01' FROM transaction; +${prepare} +`); + const blocking = await db.pool.connect(); + const concurrent = await db.pool.connect(); + // eslint-disable-next-line functional/no-try-statement + try { + await blocking.query(`BEGIN; ${blocker}`); + const expiry = db.archiveMempoolTransactions([ + { + nodeInternalId: 40001, + replacedAt: new Date('2026-01-01T01:00:00.000Z'), + transactionInternalId: 1, + }, + ]); + const waitForExpiryToBlock = async (): Promise => { + const waiting = await db.pool.query( + /* sql */ `SELECT 1 FROM pg_locks WHERE NOT granted;` + ); + if (waiting.rowCount === 0) { + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + return waitForExpiryToBlock(); + } + return undefined; + }; + await waitForExpiryToBlock(); + if (concurrently !== '') { + await concurrent.query(concurrently); + } + await blocking.query('COMMIT;'); + return await expiry; + } finally { + blocking.release(); + concurrent.release(); + } +}; + +const currentAndHistory = async (db: typeof database) => + ( + await db.pool.query<{ + replaced: string | null; + table: string; + tx: number; + }>(/* sql */ ` +SELECT 'current' AS table, transaction_internal_id::integer AS tx, + NULL AS replaced + FROM node_transaction +UNION ALL +SELECT 'history', transaction_internal_id::integer, replaced_at::text + FROM node_transaction_history +ORDER BY 1, 2;`) + ).rows; + +/* + * Regression: the advisory lock was taken inside the archival statement, whose + * snapshot predates the wait, so a block inclusion committed while it waited + * (here, block_transaction repaired for a block the node already accepted) + * was invisible to the accepted-block guard and the confirmed transaction was + * archived with its expiry. + */ +test.serial( + '[e2e] expiry sees a block inclusion committed while it waits for the expiry lock', + async (t) => { + const { db } = t.context; + const archived = await expireWhileBlocked(db, { + // another agent's expiry batch + blocker: `SELECT pg_advisory_xact_lock(${db.expiryAdvisoryLockKey});`, + concurrently: `INSERT INTO block_transaction VALUES (1, 1);`, + prepare: `INSERT INTO node_block VALUES (40001, 1, '2026-01-02');`, + }); + t.deepEqual(archived, []); + t.deepEqual(await currentAndHistory(db), [ + { replaced: null, table: 'current', tx: 1 }, + { replaced: null, table: 'current', tx: 2 }, + ]); + } +); + +/* + * Regression: an expiring parent confirmed concurrently (deleted by block + * acceptance, as by the node_block trigger) was skipped by the archival + * statement's row locks, but its snapshot still had the parent, so its + * unconfirmed child was archived with the parent's expiry. + */ +test.serial( + '[e2e] expiry leaves the child of a parent confirmed while the expiry waits', + async (t) => { + const { db } = t.context; + const archived = await expireWhileBlocked(db, { + blocker: `INSERT INTO block_transaction VALUES (1, 1); +INSERT INTO node_block VALUES (40001, 1, '2026-01-02'); +WITH confirmed AS ( + DELETE FROM node_transaction + WHERE node_internal_id = 40001 AND transaction_internal_id = 1 + RETURNING node_internal_id, transaction_internal_id, validated_at) +INSERT INTO node_transaction_history + SELECT node_internal_id, transaction_internal_id, validated_at, NULL + FROM confirmed;`, + }); + t.deepEqual(archived, []); + t.deepEqual(await currentAndHistory(db), [ + { replaced: null, table: 'current', tx: 2 }, + { replaced: null, table: 'history', tx: 1 }, + ]); + } +); + +/* + * Regression: the archival statement walked descendants in a snapshot taken + * before it waited for their row locks. For P (1) → C (2) → G (3) with only P + * expiring, C confirmed concurrently (deleted with replaced_at NULL, ending + * the cascade) was skipped by the row locks, but the walk had already reached + * G through the old C, so G was archived with P's expiry. + */ +test.serial( + '[e2e] expiry leaves the grandchild of a child confirmed while the expiry waits', + async (t) => { + const { db } = t.context; + const archived = await expireWhileBlocked(db, { + blocker: `INSERT INTO block_transaction VALUES (1, 2); +INSERT INTO node_block VALUES (40001, 1, '2026-01-02'); +WITH confirmed AS ( + DELETE FROM node_transaction + WHERE node_internal_id = 40001 AND transaction_internal_id = 2 + RETURNING node_internal_id, transaction_internal_id, validated_at) +INSERT INTO node_transaction_history + SELECT node_internal_id, transaction_internal_id, validated_at, NULL + FROM confirmed;`, + prepare: `INSERT INTO transaction VALUES (3, sha256(int8send(3::bigint))); +INSERT INTO output VALUES (sha256(int8send(3::bigint)), 0); +INSERT INTO input SELECT 3, 0, hash, 0 FROM transaction WHERE internal_id = 2; +INSERT INTO node_transaction VALUES (40001, 3, '2026-01-01');`, + }); + t.deepEqual( + archived.map((expiry) => expiry.transactionInternalId), + [1] + ); + t.deepEqual(await currentAndHistory(db), [ + { replaced: null, table: 'current', tx: 3 }, + { replaced: '2026-01-01 01:00:00', table: 'history', tx: 1 }, + { replaced: null, table: 'history', tx: 2 }, + ]); + } +); diff --git a/src/index.ts b/src/index.ts index 4149d744..449ac818 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,11 +5,24 @@ import bitcoreP2pCash from '@chaingraph/bitcore-p2p-cash'; import { Agent, cancelableDelay } from './agent.js'; import { chaingraphInternalApiPort } from './config.js'; +import { pool } from './db.js'; import { instantiateLogger } from './logging.js'; +import { + describePostgresError, + handlePoolConnectionErrors, +} from './postgres-errors.js'; const logger = instantiateLogger(); logger.info(`Starting Chaingraph...`); +handlePoolConnectionErrors(pool, (err) => { + logger.warn( + `Postgres connection lost (${describePostgresError( + err + )}); discarded it, the next query opens a new connection.` + ); +}); + const shutdownTasks = [] as (() => void)[]; const agent = new Agent({ logger, diff --git a/src/postgres-errors.spec.ts b/src/postgres-errors.spec.ts new file mode 100644 index 00000000..f0cd5f64 --- /dev/null +++ b/src/postgres-errors.spec.ts @@ -0,0 +1,68 @@ +import { EventEmitter } from 'events'; + +import test from 'ava'; +import type pg from 'pg'; + +import { + describePostgresError, + handlePoolConnectionErrors, + isPostgresConnectionError, +} from './postgres-errors.js'; + +const withCode = (code: string, message = 'error') => + Object.assign(new Error(message), { code }); + +test('isPostgresConnectionError: recognizes lost connections', (t) => { + t.true( + isPostgresConnectionError( + withCode('57P01', 'terminating connection due to administrator command') + ) + ); + t.true(isPostgresConnectionError(withCode('57P03'))); + t.true(isPostgresConnectionError(withCode('08006'))); + t.true(isPostgresConnectionError(withCode('ECONNREFUSED'))); + t.true(isPostgresConnectionError(withCode('ECONNRESET'))); + t.true( + isPostgresConnectionError(new Error('Connection terminated unexpectedly')) + ); + t.true( + isPostgresConnectionError( + new Error( + 'Client has encountered a connection error and is not queryable' + ) + ) + ); +}); + +test('isPostgresConnectionError: ignores query and data errors', (t) => { + t.false(isPostgresConnectionError(withCode('23505', 'duplicate key'))); + t.false(isPostgresConnectionError(withCode('42P01', 'missing relation'))); + t.false(isPostgresConnectionError(new Error('something else'))); + t.false(isPostgresConnectionError(undefined)); + t.false(isPostgresConnectionError('Connection terminated')); +}); + +test('describePostgresError: includes the code, omits attached state', (t) => { + const err = Object.assign(withCode('57P01', 'terminating connection'), { + client: { large: 'state' }, + }); + t.is(describePostgresError(err), '57P01: terminating connection'); + t.is(describePostgresError(new Error('plain')), 'plain'); +}); + +test('handlePoolConnectionErrors: client and pool errors do not throw', (t) => { + const pool = new EventEmitter(); + const client = new EventEmitter(); + const reported: Error[] = []; + handlePoolConnectionErrors(pool as unknown as pg.Pool, (err) => { + reported.push(err); + }); + pool.emit('connect', client); + const terminated = withCode('57P01'); + t.notThrows(() => client.emit('error', terminated)); + t.notThrows(() => + client.emit('error', new Error('Connection terminated unexpectedly')) + ); + t.notThrows(() => pool.emit('error', terminated, client)); + t.deepEqual(reported, [terminated]); +}); diff --git a/src/postgres-errors.ts b/src/postgres-errors.ts new file mode 100644 index 00000000..9db449de --- /dev/null +++ b/src/postgres-errors.ts @@ -0,0 +1,103 @@ +import type pg from 'pg'; + +/** + * Error codes Postgres sends when it closes a session: `57P01` + * (admin_shutdown – fast shutdown/restart or `pg_terminate_backend`), `57P02` + * (crash_shutdown), `57P03` (cannot_connect_now – starting up or shutting + * down). + */ +const connectionLossSqlStates = ['57P01', '57P02', '57P03']; +/** + * Socket-level errors seen while Postgres is restarting or unreachable. + */ +const connectionLossSystemErrorCodes = [ + 'ECONNREFUSED', + 'ECONNRESET', + 'EPIPE', + 'ETIMEDOUT', + 'EHOSTUNREACH', + 'ENOTFOUND', + 'EAI_AGAIN', +]; +/** + * Messages node-postgres uses for connections that ended underneath a client. + */ +const connectionLossMessages = [ + 'Connection terminated', + 'Client has encountered a connection error and is not queryable', +]; + +const isConnectionLossCode = (code: unknown) => + typeof code === 'string' && + (connectionLossSqlStates.includes(code) || + code.startsWith('08') || + connectionLossSystemErrorCodes.includes(code)); + +const isConnectionLossMessage = (message: unknown) => + typeof message === 'string' && + connectionLossMessages.some((lossMessage) => message.includes(lossMessage)); + +/** + * Returns `true` if `err` indicates the Postgres connection was lost (e.g. + * Postgres restarted), rather than a problem with the query or the data. + * Operations failing with such an error can be retried once Postgres is back. + * @param err - the error to test + */ +export const isPostgresConnectionError = (err: unknown) => { + if (typeof err !== 'object' || err === null) { + return false; + } + const { code, message } = err as { code?: unknown; message?: unknown }; + return isConnectionLossCode(code) || isConnectionLossMessage(message); +}; + +/** + * Summarize a Postgres connection error for logging without the attached + * client/socket state (node-postgres errors can carry the whole client). + * @param err - the error to describe + */ +export const describePostgresError = (err: unknown) => { + if (typeof err !== 'object' || err === null) { + return String(err); + } + const { code, message } = err as { code?: unknown; message?: unknown }; + const text = typeof message === 'string' ? message : String(err); + return typeof code === 'string' ? `${code}: ${text}` : text; +}; + +/** + * Keep a lost Postgres connection from crashing the process. + * + * When Postgres restarts (or a connection is terminated), node-postgres emits + * an `error` event: on the pool for idle clients, and on the client itself for + * checked-out clients between queries. With no listener, Node.js throws it as + * an uncaught exception. The broken client is already unusable: the pool + * discards it on release (or immediately, if idle), and the next `connect()` + * opens a fresh connection, so the only thing to do here is log it. Queries in + * flight still reject and are handled by their callers. + * @param targetPool - the pool to protect + * @param onConnectionError - called once per lost connection + */ +export const handlePoolConnectionErrors = ( + targetPool: pg.Pool, + onConnectionError: (err: Error) => void +) => { + targetPool.on('connect', (client) => { + // eslint-disable-next-line functional/no-let + let reported = false; + // a terminated client can emit more than once (error, then socket end) + client.on('error', (err) => { + if (reported) { + return; + } + reported = true; + onConnectionError(err); + }); + }); + /* + * Idle clients also emit through the pool (after the client-level listener + * above has already reported the error); this listener only prevents the + * pool's re-emitted `error` from being thrown. + */ + targetPool.on('error', () => undefined); +}; diff --git a/src/types/bitcore-lib-cash.d.ts b/src/types/bitcore-lib-cash.d.ts index fe1f9cf5..e9bb8878 100644 --- a/src/types/bitcore-lib-cash.d.ts +++ b/src/types/bitcore-lib-cash.d.ts @@ -50,6 +50,16 @@ declare module 'bitcore-lib-cash' { version: number; } + export namespace encoding { + class BufferReader { + pos: number; + + readVarintNum: () => number; + + constructor(buffer: Buffer); + } + } + export interface BitcoreBlockHeader { hash: string; prevHash: Buffer; @@ -60,9 +70,17 @@ declare module 'bitcore-lib-cash' { toHex: () => string; } + export namespace BlockHeader { + function fromBufferReader( + reader: encoding.BufferReader + ): BitcoreBlockHeader; + } + export class Transaction { toObject: () => BitcoreTransactionObject; + fromBufferReader: (reader: encoding.BufferReader) => this; + toBuffer: () => Uint8Array; isCoinbase: () => boolean; @@ -70,10 +88,12 @@ declare module 'bitcore-lib-cash' { hash: string; inputs: { - prevTxId: string; + prevTxId: Buffer; outputIndex: number; sequenceNumber: number; script: BitcoreScript; + _scriptBuffer: Buffer; + toObject: () => BitcoreTransactionObject['inputs'][number]; }[]; nLockTime: number; @@ -81,8 +101,13 @@ declare module 'bitcore-lib-cash' { outputs: { satoshis: number; script: BitcoreScript; + _scriptBuffer: Buffer; + toObject: () => BitcoreTransactionObject['outputs'][number]; tokenData?: { - amount: string; + /** + * A bn.js `BN` + */ + amount: { toString: () => string }; category: string; nft?: { capability: 'minting' | 'mutable' | 'none'; @@ -93,19 +118,28 @@ declare module 'bitcore-lib-cash' { version: number; - constructor(hexOrObject: BitcoreTransactionObjectOptions | string); + constructor(hexOrObject?: BitcoreTransactionObjectOptions | string); } export class Block { + static fromBuffer: (encoded: Buffer) => Block; + static fromObject: (contents: unknown) => Block; static fromString: (raw: string) => Block; toBuffer: () => Uint8Array; + toObject: () => unknown; + header: BitcoreBlockHeader; transactions: Transaction[]; + + constructor(contents: { + header: BitcoreBlockHeader; + transactions: Transaction[]; + }); } interface Network { diff --git a/src/types/bitcore-p2p-cash.d.ts b/src/types/bitcore-p2p-cash.d.ts index d330ace2..16172db5 100644 --- a/src/types/bitcore-p2p-cash.d.ts +++ b/src/types/bitcore-p2p-cash.d.ts @@ -23,6 +23,12 @@ declare module '@chaingraph/bitcore-p2p-cash' { Block: { fromBuffer: (rawBlock: Buffer) => BlockMessage; }; + + constructor(options?: { + Block?: typeof Block; + network?: ReturnType; + Transaction?: typeof Transaction; + }); } export const enum BitcoreInventoryType { @@ -70,6 +76,7 @@ declare module '@chaingraph/bitcore-p2p-cash' { inventory: BitcoreInventory[]; } export class SendHeadersMessage extends Message {} + export class MemPoolMessage extends Message {} export class TransactionMessage extends Message { transaction: Transaction; @@ -88,6 +95,7 @@ declare module '@chaingraph/bitcore-p2p-cash' { getdata: () => void; headers: (message: HeadersMessage) => void; inv: (message: InventoryMessage) => void; + mempool: (message: MemPoolMessage) => void; ready: () => void; tx: (message: TransactionMessage) => void; } @@ -114,6 +122,7 @@ declare module '@chaingraph/bitcore-p2p-cash' { forTransaction: (transactionHash: Buffer | string) => InventoryMessage; }; SendHeaders: typeof SendHeadersMessage; + MemPool: typeof MemPoolMessage; Transaction: typeof TransactionMessage; }; @@ -129,6 +138,7 @@ declare module '@chaingraph/bitcore-p2p-cash' { constructor(config: { host?: string; + messages?: Messages; network?: string; port?: number; relay?: boolean; diff --git a/yarn.lock b/yarn.lock index 2f10a41b..7eed91bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1362,14 +1362,14 @@ __metadata: languageName: node linkType: hard -"@types/pg@npm:^8.6.6": - version: 8.6.6 - resolution: "@types/pg@npm:8.6.6" +"@types/pg@npm:^8.20.0": + version: 8.20.0 + resolution: "@types/pg@npm:8.20.0" dependencies: "@types/node": "*" pg-protocol: "*" pg-types: ^2.2.0 - checksum: ac145553a8ad2f357feacad1bceaf5d6ce904eb9d66233b84c469a2b4fa3738d4ebdf29b7ea45387be2d07f915fd873a229f90a2f766d7c377afa7c41fbcf8d1 + checksum: e3610a00e38d52a0b9069bda48a9a5746d198f0aaa72710898fe6c277777ca3ef5190725afa61f53adb4ddd624e48267133b691f4ae7076dbef5d83aa31c2cf0 languageName: node linkType: hard @@ -2634,13 +2634,6 @@ __metadata: languageName: node linkType: hard -"buffer-writer@npm:2.0.0": - version: 2.0.0 - resolution: "buffer-writer@npm:2.0.0" - checksum: 11736b48bb75106c52ca8ec9f025e7c1b3b25ce31875f469d7210eabd5c576c329e34f6b805d4a8d605ff3f0db1e16342328802c4c963e9c826b0e43a4e631c2 - languageName: node - linkType: hard - "buffer-xor@npm:^1.0.3": version: 1.0.3 resolution: "buffer-xor@npm:1.0.3" @@ -2892,7 +2885,7 @@ __metadata: "@istanbuljs/nyc-config-typescript": ^1.0.2 "@types/dotenv": ^8.2.0 "@types/lru-cache": ^7.10.10 - "@types/pg": ^8.6.6 + "@types/pg": ^8.20.0 "@typescript-eslint/eslint-plugin": ^5.48.1 "@typescript-eslint/parser": ^5.48.1 ava: ^5.1.0 @@ -2913,7 +2906,7 @@ __metadata: npm-run-all: ^4.1.5 nyc: ^15.1.0 open-cli: ^7.1.0 - pg: ^8.8.0 + pg: ^8.22.0 pino: ^8.8.0 pino-pretty: ^9.1.1 prando: ^6.0.1 @@ -9541,13 +9534,6 @@ __metadata: languageName: node linkType: hard -"packet-reader@npm:1.0.0": - version: 1.0.0 - resolution: "packet-reader@npm:1.0.0" - checksum: 0b7516f0cbf3e322aad591bed29ba544220088c53943145c0d9121a6f59182ad811f7fd6785a8979a34356aca69d97653689029964c5998dc02645633d88ffd7 - languageName: node - linkType: hard - "pako@npm:^1.0.3, pako@npm:~1.0.5": version: 1.0.11 resolution: "pako@npm:1.0.11" @@ -9744,10 +9730,17 @@ __metadata: languageName: node linkType: hard -"pg-connection-string@npm:^2.5.0": - version: 2.5.0 - resolution: "pg-connection-string@npm:2.5.0" - checksum: a6f3a068f7c9416a5b33a326811caf0dfaaee045c225b7c628b4c9b4e9a2b25bdd12a21e4c48940e1000ea223a4e608ca122d2ff3dd08c8b1db0fc9f5705133a +"pg-cloudflare@npm:^1.4.0": + version: 1.4.0 + resolution: "pg-cloudflare@npm:1.4.0" + checksum: cddea0ab52b4c4a3c842a2304ccc10ef61e06fd00ba633f4c2c04b31e9352adee8ee0a7b1fdc8e8738527d3e39534f75903f467a0b1a6af6cc49fb5abe90481c + languageName: node + linkType: hard + +"pg-connection-string@npm:^2.14.0": + version: 2.14.0 + resolution: "pg-connection-string@npm:2.14.0" + checksum: 10e798bedffa7da43eaf6879c134c2f6e6344185e1ee3856c92444848d879ba7f0cce4981c8e3d58baf100d2da845d11b88cb9a788552d0a65bdeae71a338dce languageName: node linkType: hard @@ -9758,23 +9751,30 @@ __metadata: languageName: node linkType: hard -"pg-pool@npm:^3.5.2": - version: 3.5.2 - resolution: "pg-pool@npm:3.5.2" +"pg-pool@npm:^3.14.0": + version: 3.14.0 + resolution: "pg-pool@npm:3.14.0" peerDependencies: pg: ">=8.0" - checksum: a5d029200257671f0c17ca54b9ab6213e2060e64e5cc792b78edd50ab471a26cd364890d05d546d9296949e079e943821cf2ceb4d488f4e6a6789fb781a628bf + checksum: cf077bace1c7a55011322c71973cf22fb152bf71312c15484dc75818ace77f7759d96069fb0125163f6fa56c5470a0ea70734c2f6218d398018e4ecf835bf40e languageName: node linkType: hard -"pg-protocol@npm:*, pg-protocol@npm:^1.5.0": +"pg-protocol@npm:*": version: 1.5.0 resolution: "pg-protocol@npm:1.5.0" checksum: b839d12cafe942ef9cbc5b13c174eb2356804fb4fe8ead8279f46a36be90722d19a91409955beb8a3d5301639c44854e49749de4aef02dc361fee3e2a61fb1e4 languageName: node linkType: hard -"pg-types@npm:^2.1.0, pg-types@npm:^2.2.0": +"pg-protocol@npm:^1.15.0": + version: 1.15.0 + resolution: "pg-protocol@npm:1.15.0" + checksum: eef533572ab655dada3a897646194d81b4901bd1729b134c2d5187cc617f71d1a77465efb6ab825486e35cbb8a0311f7ad3893f60e16c9707c0e31f05373a7c3 + languageName: node + linkType: hard + +"pg-types@npm:2.2.0, pg-types@npm:^2.2.0": version: 2.2.0 resolution: "pg-types@npm:2.2.0" dependencies: @@ -9787,27 +9787,29 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.8.0": - version: 8.8.0 - resolution: "pg@npm:8.8.0" - dependencies: - buffer-writer: 2.0.0 - packet-reader: 1.0.0 - pg-connection-string: ^2.5.0 - pg-pool: ^3.5.2 - pg-protocol: ^1.5.0 - pg-types: ^2.1.0 - pgpass: 1.x +"pg@npm:^8.22.0": + version: 8.22.0 + resolution: "pg@npm:8.22.0" + dependencies: + pg-cloudflare: ^1.4.0 + pg-connection-string: ^2.14.0 + pg-pool: ^3.14.0 + pg-protocol: ^1.15.0 + pg-types: 2.2.0 + pgpass: 1.0.5 peerDependencies: pg-native: ">=3.0.1" + dependenciesMeta: + pg-cloudflare: + optional: true peerDependenciesMeta: pg-native: optional: true - checksum: fa30a85814dd7238b582c3bc6c0b9e2b0ae38dd0a6bb485ef480e64bb5ce589de6cb873ce4d3cd10c37a3e0a1e1281ba75dc7d80b1a68bae91999cd5b70d398b + checksum: 68695b3427476bc5a8838fddb4ab56f091319d74d45ee20c3e9fc1414c848bc9535ef4b53977c4407c511296a1848a543ba7824bd4652fd9c146e0e04bc4935b languageName: node linkType: hard -"pgpass@npm:1.x": +"pgpass@npm:1.0.5": version: 1.0.5 resolution: "pgpass@npm:1.0.5" dependencies: