diff --git a/.github/workflows/internal-build.yml b/.github/workflows/internal-build.yml index fe9ad782..f7e9ef8b 100644 --- a/.github/workflows/internal-build.yml +++ b/.github/workflows/internal-build.yml @@ -305,6 +305,7 @@ jobs: args="$(<<< $image_json jq -c -r '[ .deps[] | [ "--build-arg \(.name | ascii_upcase)_IMAGE=stellar-\(.name):\(.sha)-${{ matrix.arch }}" ] ] | flatten | join(" ")')" + args="${args} $(<<< $image_json jq -r '.config.thruput // empty | "--build-arg VARIANT_IMAGE=thruput --build-arg THRUPUT_VERSION=\(.cli) --build-arg HOG_DEVNODE_VERSION=\(.hog_devnode)"')" echo "args=${args}" | tee -a $GITHUB_OUTPUT - name: Build Image run: > diff --git a/Dockerfile b/Dockerfile index f279a02e..f32895b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ ARG FRIENDBOT_IMAGE=stellar-friendbot-stage ARG RPC_IMAGE=stellar-rpc-stage ARG LAB_IMAGE=stellar-lab-stage ARG GALEXIE_IMAGE=stellar-galexie-stage +ARG VARIANT_IMAGE=quickstart # xdr @@ -206,6 +207,21 @@ FROM scratch AS stellar-galexie-stage COPY --from=stellar-galexie-builder /galexie /galexie +# thruput + +# The Thruput CLI is built from its crates.io release on the same base as the +# final image, because Rust images ship a newer glibc. The CLI then installs +# the published hog-devnode runtime it supervises, verifying its checksum. +FROM ubuntu:24.04 AS thruput-builder +ARG THRUPUT_VERSION +ARG HOG_DEVNODE_VERSION +RUN apt-get update && apt-get install -y curl ca-certificates build-essential pkg-config libssl-dev +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal +ENV PATH="/root/.cargo/bin:$PATH" +RUN cargo install thruput --locked --version "$THRUPUT_VERSION" --root /thruput +RUN /thruput/bin/thruput toolchain install "$HOG_DEVNODE_VERSION" +RUN cp -r /root/.cache/thruput/toolchain/hog-devnode/"$HOG_DEVNODE_VERSION"/*-linux-gnu /hog-devnode + # quickstart FROM $XDR_IMAGE AS xdr @@ -266,3 +282,15 @@ ADD start / RUN ["chmod", "+x", "start"] ENTRYPOINT ["/start"] + +# The thruput image variant adds the Thruput CLI, its hog-devnode runtime and +# the shared libraries hog-devnode needs, and the Thruput service config. +FROM quickstart AS thruput +COPY --from=thruput-builder /thruput/bin/thruput /usr/local/bin/thruput +COPY --from=thruput-builder /hog-devnode /usr/local/lib/hog-devnode +RUN apt-retry sh -c 'apt-get update && apt-get install -y libatomic1 libjemalloc2 libsnappy1v5 liblz4-1 libzstd1' && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +COPY thruput /opt/stellar-default/thruput + +# The image built is quickstart, or the variant of it named by VARIANT_IMAGE. +FROM $VARIANT_IMAGE diff --git a/Makefile b/Makefile index 1c9a5e1c..1d8835c1 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,12 @@ FRIENDBOT_OPTIONS = $(shell < $(IMAGE_JSON) jq -c '.deps[] | select(.name == "fr LAB_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "lab") | .repo') LAB_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "lab") | .sha') +# Images with a thruput config also install the Thruput CLI and its local node +# runtime, both published releases. +THRUPUT_VERSION = $(shell < $(IMAGE_JSON) jq -r '.config.thruput.cli // empty') +HOG_DEVNODE_VERSION = $(shell < $(IMAGE_JSON) jq -r '.config.thruput.hog_devnode // empty') +THRUPUT_BUILD_ARGS = $(if $(THRUPUT_VERSION),--build-arg VARIANT_IMAGE=thruput --build-arg THRUPUT_VERSION="$(THRUPUT_VERSION)" --build-arg HOG_DEVNODE_VERSION="$(HOG_DEVNODE_VERSION)") + run: $(CONTAINER_RUNTIME) run --rm -i --name stellar -p 8000:8000 -p 11626:11626 stellar/quickstart:$(TAG) --local @@ -54,7 +60,8 @@ build: $(IMAGE_JSON) --build-arg GALEXIE_REPO="$(GALEXIE_REPO)" --build-arg GALEXIE_REF="$(GALEXIE_SHA)" \ --build-arg HORIZON_REPO="$(HORIZON_REPO)" --build-arg HORIZON_REF="$(HORIZON_SHA)" --build-arg HORIZON_OPTIONS='$(HORIZON_OPTIONS)' \ --build-arg FRIENDBOT_REPO="$(FRIENDBOT_REPO)" --build-arg FRIENDBOT_REF="$(FRIENDBOT_SHA)" --build-arg FRIENDBOT_OPTIONS='$(FRIENDBOT_OPTIONS)' \ - --build-arg LAB_REPO="$(LAB_REPO)" --build-arg LAB_REF=$(LAB_SHA) + --build-arg LAB_REPO="$(LAB_REPO)" --build-arg LAB_REF=$(LAB_SHA) \ + $(THRUPUT_BUILD_ARGS) # Run the same tests that CI runs against a running quickstart container. # Build and run the container first with: make build run TAG=... diff --git a/README.md b/README.md index e65d3205..1c301978 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ HTTP APIs and Tools are available at the following port and paths: - Ledger Meta: `http://localhost:8000/ledger-meta` (available with `--local` and `--enable galexie`) - History Archive: `http://localhost:8000/archive` (available with `--local` only) +The `thruput` image also runs a local Thruput network, see [Thruput](#thruput). + ## Tags The following image tags are available on DockerHub: @@ -299,6 +301,57 @@ The ledger meta store includes: - `.config.json` - Configuration file describing the data format - Partition directories containing compressed XDR files compressed with zstd with one ledger per file (`.xdr.zst`) +### Thruput + +The `thruput` image also runs a local [Thruput](https://www.thruput.network) network alongside the local Stellar network. It is built from Thruput's public releases: the [`thruput`](https://crates.io/crates/thruput) CLI from crates.io, and the `hog-devnode` runtime that the CLI installs and verifies. It adds these services: + +- `thruput-node` - a local Thruput network, `hog-devnode` supervised by `thruput node`, which serves the Thruput [JSON-RPC API](https://www.thruput.network/docs/reference/json-rpc-api/), including the local faucet. +- `thruput-accounts` - creates a Thruput account for every account created on the Stellar network, for the same ed25519 key, funded by an airdrop from the faucet. It follows account creations in Horizon, so it runs only when `horizon` is enabled too. + +The `thruput` image uses `--local` network mode by default. The services run only in that mode, where they are enabled by default. To run without them, use the `--enable` option to specify the services you wish to run without `thruput`, for example `--enable core,horizon,rpc`. + +Build the image with: + +``` +make build TAG=thruput +``` + +The Thruput versions are set in the `thruput` image's `config` in `images.json`. + +Run it with: + +``` +docker run --rm -it -p 8000:8000 --name stellar stellar/quickstart:thruput +``` + +The Thruput RPC is available at `http://localhost:8000/thruput/rpc`. + +To use the network with the [`thruput` CLI](https://www.thruput.network/docs/cli/install/), create a profile for it: + +``` +thruput config new quickstart --network local --rpc-url http://localhost:8000/thruput/rpc --use --global +``` + +The CLI funds accounts on a local network with the faucet named in the node's runtime manifest, so copy the manifest out of the container, and pass it to `thruput airdrop`. The manifest changes when the container restarts, so copy it again after each restart. Create a funded key with: + +``` +docker cp stellar:/opt/stellar/thruput/node/data/runtime/runtime-manifest.json quickstart-manifest.json +thruput keys generate alice +thruput airdrop alice --runtime-manifest quickstart-manifest.json +thruput balance alice +``` + +The key of every Stellar account also has a Thruput account. To use a Stellar key with the `thruput` CLI, import its secret. `thruput keys ls` shows the address of its Thruput account, and `thruput balance` its balance: + +``` +stellar keys generate carol --network local --fund +stellar keys secret carol | thruput keys import carol +thruput keys ls +thruput balance carol +``` + +The hosted Thruput [explorer](https://www.thruput.network/docs/tools/block-explorer/) and [faucet](https://www.thruput.network/docs/tools/faucet/) serve Futurenet, and are not included in the image. + ### Using in GitHub Actions The quickstart image can be run in GitHub Actions workflows using the provided action. This is useful for testing smart contracts, running integration tests, or any other CI/CD workflows that need a Stellar network. diff --git a/images.json b/images.json index 0266976b..20fe260d 100644 --- a/images.json +++ b/images.json @@ -331,5 +331,20 @@ "continue-on-error": true, "additional-tests": [] } + }, + { + "tag": "thruput", + "inherit": "latest", + "events": [], + "config": { + "thruput": { + "cli": "0.42.0", + "hog_devnode": "0.1.31" + } + }, + "tests": { + "continue-on-error": false, + "additional-tests": [] + } } ] diff --git a/start b/start index 6a7cc80d..7adec302 100755 --- a/start +++ b/start @@ -6,6 +6,7 @@ green=$(echo -e "\e[32m") blue=$(echo -e "\e[34m") purple=$(echo -e "\e[35m") cyan=$(echo -e "\e[36m") +yellow=$(echo -e "\e[33m") export STELLAR_HOME="/opt/stellar" export PGHOME="$STELLAR_HOME/postgresql" @@ -19,6 +20,7 @@ export STELLAR_RPC_HOME="$STELLAR_HOME/stellar-rpc" export GALEXIEHOME="$STELLAR_HOME/galexie" export LEDGERMETASTOREHOME="$STELLAR_HOME/ledger-meta-store" export HISTORYARCHIVEHOME="$STELLAR_HOME/history-archive" +export THRUPUTHOME="$STELLAR_HOME/thruput" export CORELOG="/var/log/stellar-core" @@ -30,12 +32,13 @@ export PGPORT=5432 export PROTOCOL_VERSION_DEFAULT="$(< /image.json jq -r '.config.protocol_version_default')" : "${PROTOCOL_VERSION:=$PROTOCOL_VERSION_DEFAULT}" -: "${ENABLE:=core,horizon,rpc}" +: "${ENABLE:=core,horizon,rpc,thruput}" : "${ENABLE_LOGS:=false}" : "${ENABLE_CORE:=false}" : "${ENABLE_HORIZON:=false}" : "${ENABLE_LAB:=false}" : "${ENABLE_GALEXIE:=false}" +: "${ENABLE_THRUPUT:=false}" # TODO: Remove once the Soroban RPC name is fully deprecated : "${ENABLE_SOROBAN_RPC:=false}" : "${ENABLE_RPC:=$ENABLE_SOROBAN_RPC}" @@ -105,6 +108,11 @@ function start() { echo " $(< /image.json jq -r '.deps[] | select(.name == "lab") | "\(.ref) (\(.sha))"')" echo " galexie:" echo " $(< /image.json jq -r '.deps[] | select(.name == "galexie") | "\(.ref) (\(.sha))"')" + if thruput_installed; then + echo " thruput:" + echo " cli: $(< /image.json jq -r '.config.thruput.cli')" + echo " hog-devnode: $(< /image.json jq -r '.config.thruput.hog_devnode')" + fi echo "mode: $STELLAR_MODE" echo "network: $NETWORK" @@ -124,6 +132,7 @@ function start() { copy_pgpass init_stellar_rpc init_galexie + init_thruput stop_postgres # this gets started in init_db @@ -216,7 +225,12 @@ function process_args() { # TODO: ask for what network to use if [ -z "$NETWORK" ]; then - NETWORK="testnet" + # The thruput image runs the local network by default, where Thruput runs. + if thruput_installed; then + NETWORK="local" + else + NETWORK="testnet" + fi fi if [[ ",$ENABLE," = *",core,"* ]]; then @@ -239,6 +253,16 @@ function process_args() { ENABLE_GALEXIE=true fi + if [[ ",$ENABLE," = *",thruput,"* ]]; then + ENABLE_THRUPUT=true + fi + + # Thruput runs only in the thruput image's local network, and is ignored + # elsewhere. + if [ "$NETWORK" != "local" ] || ! thruput_installed; then + ENABLE_THRUPUT=false + fi + case "$NETWORK" in testnet) export NETWORK_PASSPHRASE="Test SDF Network ; September 2015" @@ -344,6 +368,9 @@ function copy_defaults() { cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/ledger-meta-store.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/friendbot.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/history-archive.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true + if thruput_installed; then + cp /opt/stellar-default/thruput/supervisor/etc/supervisord.conf.d/*.conf $SUPHOME/etc/supervisord.conf.d + fi fi if [ -d $COREHOME/etc ]; then @@ -408,6 +435,17 @@ function copy_defaults() { if [ -d /opt/stellar-default/$NETWORK/nginx/ ]; then $CP /opt/stellar-default/$NETWORK/nginx/ $NXHOME fi + if thruput_installed; then + $CP /opt/stellar-default/thruput/nginx/ $NXHOME + fi + fi + + if thruput_installed; then + if [ -d $THRUPUTHOME ]; then + echo "thruput: config directory exists, skipping copy" + else + $CP /opt/stellar-default/thruput/ $THRUPUTHOME + fi fi if [ -d $GALEXIEHOME/etc ]; then @@ -658,6 +696,18 @@ function init_galexie() { popd } +function thruput_installed() { + [ -x /usr/local/bin/thruput ] +} + +function init_thruput() { + if [ "$ENABLE_THRUPUT" != "true" ]; then + return 0 + fi + + run_silent "chown-thruput" chown -R stellar:stellar $THRUPUTHOME +} + function kill_supervisor() { kill -3 $(cat "/var/run/supervisord.pid") } @@ -896,6 +946,13 @@ function start_optional_services() { supervisorctl start ledger-meta-store supervisorctl start galexie fi + + if [ "$ENABLE_THRUPUT" == "true" ]; then + supervisorctl start thruput-node + if [ "$ENABLE_HORIZON" == "true" ]; then + supervisorctl start thruput-accounts + fi + fi } function exec_supervisor() { @@ -928,6 +985,9 @@ function print_service_logs() { if [ "$ENABLE_RPC" = "true" ]; then supervisorctl tail -f stellar-rpc stdout > >(sed "s/^/${blue}stellar-rpc | $clear/") & fi + if [ "$ENABLE_THRUPUT" = "true" ]; then + supervisorctl tail -f thruput-node stdout > >(sed "s/^/${yellow}thruput-node | $clear/") & + fi } # run_silent is a utility function that runs a command with an abbreviated @@ -1015,6 +1075,9 @@ function service_status() { stellar_core_status "rpc" 11826 & stellar_rpc_status & fi + if [ "$ENABLE_THRUPUT" = "true" ]; then + thruput_status & + fi } function stellar_core_status() { @@ -1048,6 +1111,19 @@ function stellar_rpc_status() { echo "stellar-rpc: up and ready" } +function thruput_status() { + echo "thruput: waiting for ready state..." + while ! curl --silent --fail --request POST 'http://localhost:8000/thruput/rpc' \ + --header 'Content-Type: application/json' \ + --data-raw '{ "jsonrpc": "2.0", "id": 1, "method": "getChainTip" }' | + jq --exit-status '.result != null' > /dev/null 2>&1; + do + sleep 5 + done + echo "thruput: up and ready" + echo "thruput: rpc: http://localhost:8000/thruput/rpc" +} + function horizon_status() { COUNTER=1 echo "horizon: waiting for ingestion to catch up..." diff --git a/thruput/accounts/bin/start b/thruput/accounts/bin/start new file mode 100755 index 00000000..4152abf2 --- /dev/null +++ b/thruput/accounts/bin/start @@ -0,0 +1,99 @@ +#! /usr/bin/env python3 + +# Creates a Thruput account for every account created on the Stellar network, +# for the same ed25519 key. It follows the account_created effects in Horizon, +# and creates each Thruput account by requesting an airdrop for it from the +# local node's faucet. The last effect handled is kept in the file cursor, so a +# restart continues where it stopped. + +import base64 +import hashlib +import json +import time +import urllib.request + +HORIZON = "http://localhost:8001" +THRUPUT_RPC = "http://127.0.0.1:9545" +BASE58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + + +def base58(data): + n = int.from_bytes(data, "big") + out = "" + while n: + n, r = divmod(n, 58) + out = BASE58[r] + out + return "1" * (len(data) - len(data.lstrip(b"\0"))) + out + + +def thruput_address(stellar_account): + # A Stellar account ID is a G StrKey of an ed25519 public key. The + # Thruput account of that key is its pubkey account with nonce 0, whose + # address is sha256(cxs(DomainSeparator("PubkeyAccount")) || + # cxs(PubkeyAccountPreimage)), where the preimage names the key by + # sha256(cxs(DomainSeparator("PublicKey")) || cxs(PublicKey::Ed25519)). + key = base64.b32decode(stellar_account)[1:33] + key_hash = hashlib.sha256(b"\x00\x09PublicKey" + b"\x00" + key).digest() + address = hashlib.sha256(b"\x00\x0dPubkeyAccount" + key_hash + b"\x00").digest() + return base58(address) + + +def post(url, body): + request = urllib.request.Request( + url, json.dumps(body).encode(), {"Content-Type": "application/json"} + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + +def create_thruput_account(stellar_account): + address = thruput_address(stellar_account) + # The airdrop is keyed by the Stellar key, so requesting it until it + # succeeds funds the account only once. + nonce = "0x" + base64.b32decode(stellar_account)[1:33].hex() + request = { + "jsonrpc": "2.0", + "id": 1, + "method": "faucet_requestAirdrop", + "params": {"recipient": address, "clientNonce": nonce}, + } + while True: + try: + result = post(THRUPUT_RPC, request).get("result") or {} + if result.get("attemptStatus") == "succeeded": + print(f"thruput account {address} for stellar account " + f"{stellar_account}: {result['transactionHash']}", flush=True) + return + except OSError: + pass + time.sleep(1) + + +def main(): + try: + with open("cursor") as f: + cursor = f.read().strip() + except FileNotFoundError: + cursor = "" + print("creating thruput accounts for new stellar accounts...", flush=True) + while True: + url = f"{HORIZON}/effects?order=asc&limit=200" + if cursor: + url += f"&cursor={cursor}" + try: + with urllib.request.urlopen(url, timeout=30) as response: + records = json.load(response)["_embedded"]["records"] + except OSError: + records = [] + if not records: + time.sleep(5) + continue + for record in records: + if record["type"] == "account_created": + create_thruput_account(record["account"]) + cursor = record["paging_token"] + with open("cursor", "w") as f: + f.write(cursor) + + +main() diff --git a/thruput/nginx/etc/conf.d/thruput.conf b/thruput/nginx/etc/conf.d/thruput.conf new file mode 100644 index 00000000..e54fc4ff --- /dev/null +++ b/thruput/nginx/etc/conf.d/thruput.conf @@ -0,0 +1,6 @@ +location /thruput/rpc { + rewrite ^/thruput/rpc/?(.*)$ /$1 break; + proxy_set_header Host $http_host; + proxy_pass http://127.0.0.1:9545; + proxy_redirect off; +} diff --git a/thruput/node/bin/start b/thruput/node/bin/start new file mode 100755 index 00000000..cf2a5082 --- /dev/null +++ b/thruput/node/bin/start @@ -0,0 +1,22 @@ +#! /usr/bin/env bash + +set -e + +# Runs the local Thruput network: hog-devnode, supervised by `thruput node`, +# which serves its JSON-RPC API, including the local faucet, on +# 127.0.0.1:9545. nginx serves it at /thruput/rpc. +# +# No key is funded at genesis; accounts are funded through the faucet. The RAM +# database is capped well below its default to suit Docker hosts with less +# memory. + +export HOME=/opt/stellar/thruput/node + +echo "starting thruput node..." +exec thruput node \ + --offline \ + --hog-devnode-binary /usr/local/lib/hog-devnode/hog-devnode \ + --host 127.0.0.1 \ + --port 9545 \ + --ramdb-bytes 2G \ + --data-dir "$HOME/data" diff --git a/thruput/supervisor/etc/supervisord.conf.d/thruput-accounts.conf b/thruput/supervisor/etc/supervisord.conf.d/thruput-accounts.conf new file mode 100644 index 00000000..28a4b0ef --- /dev/null +++ b/thruput/supervisor/etc/supervisord.conf.d/thruput-accounts.conf @@ -0,0 +1,9 @@ +[program:thruput-accounts] +user=stellar +directory=/opt/stellar/thruput/accounts +command=/opt/stellar/thruput/accounts/bin/start +autostart=false +autorestart=true +priority=84 +startretries=50 +redirect_stderr=true diff --git a/thruput/supervisor/etc/supervisord.conf.d/thruput-node.conf b/thruput/supervisor/etc/supervisord.conf.d/thruput-node.conf new file mode 100644 index 00000000..0aa0d98a --- /dev/null +++ b/thruput/supervisor/etc/supervisord.conf.d/thruput-node.conf @@ -0,0 +1,8 @@ +[program:thruput-node] +user=stellar +directory=/opt/stellar/thruput/node +command=/opt/stellar/thruput/node/bin/start +autostart=false +autorestart=true +priority=80 +redirect_stderr=true