Add thruput

.github/workflows/internal-build.yml

@@ -305,6 +305,7 @@ jobs:
         args="$(<<< $image_json jq -c -r '[ .deps[] | [
         "--build-arg \(.name | ascii_upcase)_IMAGE=stellar-\(.name):\(.sha)-${{ matrix.arch }}"
         ] ] | flatten | join(" ")')"
+        args="${args} $(<<< $image_json jq -r '.config.thruput // empty | "--build-arg VARIANT_IMAGE=thruput --build-arg THRUPUT_VERSION=\(.cli) --build-arg HOG_DEVNODE_VERSION=\(.hog_devnode)"')"
         echo "args=${args}" | tee -a $GITHUB_OUTPUT
     - name: Build Image
       run: >

Dockerfile

@@ -19,6 +19,7 @@ ARG FRIENDBOT_IMAGE=stellar-friendbot-stage
 ARG RPC_IMAGE=stellar-rpc-stage
 ARG LAB_IMAGE=stellar-lab-stage
 ARG GALEXIE_IMAGE=stellar-galexie-stage
+ARG VARIANT_IMAGE=quickstart
 
 # xdr
 
@@ -206,6 +207,21 @@ FROM scratch AS stellar-galexie-stage
 
 COPY --from=stellar-galexie-builder /galexie /galexie
 
+# thruput
+
+# The Thruput CLI is built from its crates.io release on the same base as the
+# final image, because Rust images ship a newer glibc. The CLI then installs
+# the published hog-devnode runtime it supervises, verifying its checksum.
+FROM ubuntu:24.04 AS thruput-builder
+ARG THRUPUT_VERSION
+ARG HOG_DEVNODE_VERSION
+RUN apt-get update && apt-get install -y curl ca-certificates build-essential pkg-config libssl-dev
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
+ENV PATH="/root/.cargo/bin:$PATH"
+RUN cargo install thruput --locked --version "$THRUPUT_VERSION" --root /thruput
+RUN /thruput/bin/thruput toolchain install "$HOG_DEVNODE_VERSION"
+RUN cp -r /root/.cache/thruput/toolchain/hog-devnode/"$HOG_DEVNODE_VERSION"/*-linux-gnu /hog-devnode
+
 # quickstart
 
 FROM $XDR_IMAGE AS xdr
@@ -266,3 +282,15 @@ ADD start /
 RUN ["chmod", "+x", "start"]
 
 ENTRYPOINT ["/start"]
+
+# The thruput image variant adds the Thruput CLI, its hog-devnode runtime and
+# the shared libraries hog-devnode needs, and the Thruput service config.
+FROM quickstart AS thruput
+COPY --from=thruput-builder /thruput/bin/thruput /usr/local/bin/thruput
+COPY --from=thruput-builder /hog-devnode /usr/local/lib/hog-devnode
+RUN apt-retry sh -c 'apt-get update && apt-get install -y libatomic1 libjemalloc2 libsnappy1v5 liblz4-1 libzstd1' && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY thruput /opt/stellar-default/thruput
+
+# The image built is quickstart, or the variant of it named by VARIANT_IMAGE.
+FROM $VARIANT_IMAGE

Makefile

@@ -36,6 +36,12 @@ FRIENDBOT_OPTIONS = $(shell < $(IMAGE_JSON) jq -c '.deps[] | select(.name == "fr
 LAB_REPO =          $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "lab") | .repo')
 LAB_SHA =           $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "lab") | .sha')
 
+# Images with a thruput config also install the Thruput CLI and its local node
+# runtime, both published releases.
+THRUPUT_VERSION =     $(shell < $(IMAGE_JSON) jq -r '.config.thruput.cli // empty')
+HOG_DEVNODE_VERSION = $(shell < $(IMAGE_JSON) jq -r '.config.thruput.hog_devnode // empty')
+THRUPUT_BUILD_ARGS =  $(if $(THRUPUT_VERSION),--build-arg VARIANT_IMAGE=thruput --build-arg THRUPUT_VERSION="$(THRUPUT_VERSION)" --build-arg HOG_DEVNODE_VERSION="$(HOG_DEVNODE_VERSION)")
+
 run:
 	$(CONTAINER_RUNTIME) run --rm -i --name stellar -p 8000:8000 -p 11626:11626 stellar/quickstart:$(TAG) --local
 
@@ -54,7 +60,8 @@ build: $(IMAGE_JSON)
 		--build-arg GALEXIE_REPO="$(GALEXIE_REPO)" --build-arg GALEXIE_REF="$(GALEXIE_SHA)" \
 		--build-arg HORIZON_REPO="$(HORIZON_REPO)" --build-arg HORIZON_REF="$(HORIZON_SHA)" --build-arg HORIZON_OPTIONS='$(HORIZON_OPTIONS)' \
 		--build-arg FRIENDBOT_REPO="$(FRIENDBOT_REPO)" --build-arg FRIENDBOT_REF="$(FRIENDBOT_SHA)" --build-arg FRIENDBOT_OPTIONS='$(FRIENDBOT_OPTIONS)' \
-		--build-arg LAB_REPO="$(LAB_REPO)" --build-arg LAB_REF=$(LAB_SHA)
+		--build-arg LAB_REPO="$(LAB_REPO)" --build-arg LAB_REF=$(LAB_SHA) \
+		$(THRUPUT_BUILD_ARGS)
 
 # Run the same tests that CI runs against a running quickstart container.
 # Build and run the container first with: make build run TAG=...

README.md

@@ -61,6 +61,8 @@ HTTP APIs and Tools are available at the following port and paths:
 - Ledger Meta: `http://localhost:8000/ledger-meta` (available with `--local` and `--enable galexie`)
 - History Archive: `http://localhost:8000/archive` (available with `--local` only)
 
+The `thruput` image also runs a local Thruput network, see [Thruput](#thruput).
+
 ## Tags
 
 The following image tags are available on DockerHub:
@@ -299,6 +301,57 @@ The ledger meta store includes:
 - `.config.json` - Configuration file describing the data format
 - Partition directories containing compressed XDR files compressed with zstd with one ledger per file (`.xdr.zst`)
 
+### Thruput
+
+The `thruput` image also runs a local [Thruput](https://www.thruput.network) network alongside the local Stellar network. It is built from Thruput's public releases: the [`thruput`](https://crates.io/crates/thruput) CLI from crates.io, and the `hog-devnode` runtime that the CLI installs and verifies. It adds these services:
+
+- `thruput-node` - a local Thruput network, `hog-devnode` supervised by `thruput node`, which serves the Thruput [JSON-RPC API](https://www.thruput.network/docs/reference/json-rpc-api/), including the local faucet.
+- `thruput-accounts` - creates a Thruput account for every account created on the Stellar network, for the same ed25519 key, funded by an airdrop from the faucet. It follows account creations in Horizon, so it runs only when `horizon` is enabled too.
+
+The `thruput` image uses `--local` network mode by default. The services run only in that mode, where they are enabled by default. To run without them, use the `--enable` option to specify the services you wish to run without `thruput`, for example `--enable core,horizon,rpc`.
+
+Build the image with:
+
+```
+make build TAG=thruput
+```
+
+The Thruput versions are set in the `thruput` image's `config` in `images.json`.
+
+Run it with:
+
+```
+docker run --rm -it -p 8000:8000 --name stellar stellar/quickstart:thruput
+```
+
+The Thruput RPC is available at `http://localhost:8000/thruput/rpc`.
+
+To use the network with the [`thruput` CLI](https://www.thruput.network/docs/cli/install/), create a profile for it:
+
+```
+thruput config new quickstart --network local --rpc-url http://localhost:8000/thruput/rpc --use --global
+```
+
+The CLI funds accounts on a local network with the faucet named in the node's runtime manifest, so copy the manifest out of the container, and pass it to `thruput airdrop`. The manifest changes when the container restarts, so copy it again after each restart. Create a funded key with:
+
+```
+docker cp stellar:/opt/stellar/thruput/node/data/runtime/runtime-manifest.json quickstart-manifest.json
+thruput keys generate alice
+thruput airdrop alice --runtime-manifest quickstart-manifest.json
+thruput balance alice
+```
+
+The key of every Stellar account also has a Thruput account. To use a Stellar key with the `thruput` CLI, import its secret. `thruput keys ls` shows the address of its Thruput account, and `thruput balance` its balance:
+
+```
+stellar keys generate carol --network local --fund
+stellar keys secret carol | thruput keys import carol
+thruput keys ls
+thruput balance carol
+```
+
+The hosted Thruput [explorer](https://www.thruput.network/docs/tools/block-explorer/) and [faucet](https://www.thruput.network/docs/tools/faucet/) serve Futurenet, and are not included in the image.
+
 ### Using in GitHub Actions
 
 The quickstart image can be run in GitHub Actions workflows using the provided action. This is useful for testing smart contracts, running integration tests, or any other CI/CD workflows that need a Stellar network.

images.json

@@ -331,5 +331,20 @@
       "continue-on-error": true,
       "additional-tests": []
     }
+  },
+  {
+    "tag": "thruput",
+    "inherit": "latest",
+    "events": [],
+    "config": {
+      "thruput": {
+        "cli": "0.42.0",
+        "hog_devnode": "0.1.31"
+      }
+    },
+    "tests": {
+      "continue-on-error": false,
+      "additional-tests": []
+    }
   }
 ]

start

@@ -6,6 +6,7 @@ green=$(echo -e "\e[32m")
 blue=$(echo -e "\e[34m")
 purple=$(echo -e "\e[35m")
 cyan=$(echo -e "\e[36m")
+yellow=$(echo -e "\e[33m")
 
 export STELLAR_HOME="/opt/stellar"
 export PGHOME="$STELLAR_HOME/postgresql"
@@ -19,6 +20,7 @@ export STELLAR_RPC_HOME="$STELLAR_HOME/stellar-rpc"
 export GALEXIEHOME="$STELLAR_HOME/galexie"
 export LEDGERMETASTOREHOME="$STELLAR_HOME/ledger-meta-store"
 export HISTORYARCHIVEHOME="$STELLAR_HOME/history-archive"
+export THRUPUTHOME="$STELLAR_HOME/thruput"
 
 export CORELOG="/var/log/stellar-core"
 
@@ -30,12 +32,13 @@ export PGPORT=5432
 export PROTOCOL_VERSION_DEFAULT="$(< /image.json jq -r '.config.protocol_version_default')"
 
 : "${PROTOCOL_VERSION:=$PROTOCOL_VERSION_DEFAULT}"
-: "${ENABLE:=core,horizon,rpc}"
+: "${ENABLE:=core,horizon,rpc,thruput}"
 : "${ENABLE_LOGS:=false}"
 : "${ENABLE_CORE:=false}"
 : "${ENABLE_HORIZON:=false}"
 : "${ENABLE_LAB:=false}"
 : "${ENABLE_GALEXIE:=false}"
+: "${ENABLE_THRUPUT:=false}"
 # TODO: Remove once the Soroban RPC name is fully deprecated
 : "${ENABLE_SOROBAN_RPC:=false}"
 : "${ENABLE_RPC:=$ENABLE_SOROBAN_RPC}"
@@ -105,6 +108,11 @@ function start() {
   echo "    $(< /image.json jq -r '.deps[] | select(.name == "lab") | "\(.ref) (\(.sha))"')"
   echo "  galexie:"
   echo "    $(< /image.json jq -r '.deps[] | select(.name == "galexie") | "\(.ref) (\(.sha))"')"
+  if thruput_installed; then
+    echo "  thruput:"
+    echo "    cli: $(< /image.json jq -r '.config.thruput.cli')"
+    echo "    hog-devnode: $(< /image.json jq -r '.config.thruput.hog_devnode')"
+  fi
 
   echo "mode: $STELLAR_MODE"
   echo "network: $NETWORK"
@@ -124,6 +132,7 @@ function start() {
   copy_pgpass
   init_stellar_rpc
   init_galexie
+  init_thruput
 
   stop_postgres  # this gets started in init_db
 
@@ -216,7 +225,12 @@ function process_args() {
 
   # TODO: ask for what network to use
   if [ -z "$NETWORK" ]; then
-    NETWORK="testnet"
+    # The thruput image runs the local network by default, where Thruput runs.
+    if thruput_installed; then
+      NETWORK="local"
+    else
+      NETWORK="testnet"
+    fi
   fi
 
   if [[ ",$ENABLE," = *",core,"* ]]; then
@@ -239,6 +253,16 @@ function process_args() {
     ENABLE_GALEXIE=true
   fi
 
+  if [[ ",$ENABLE," = *",thruput,"* ]]; then
+    ENABLE_THRUPUT=true
+  fi
+
+  # Thruput runs only in the thruput image's local network, and is ignored
+  # elsewhere.
+  if [ "$NETWORK" != "local" ] || ! thruput_installed; then
+    ENABLE_THRUPUT=false
+  fi
+
   case "$NETWORK" in
   testnet)
     export NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
@@ -344,6 +368,9 @@ function copy_defaults() {
     cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/ledger-meta-store.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true
     cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/friendbot.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true
     cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/history-archive.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true
+    if thruput_installed; then
+      cp /opt/stellar-default/thruput/supervisor/etc/supervisord.conf.d/*.conf $SUPHOME/etc/supervisord.conf.d
+    fi
   fi
 
   if [ -d $COREHOME/etc ]; then
@@ -408,6 +435,17 @@ function copy_defaults() {
     if [ -d /opt/stellar-default/$NETWORK/nginx/ ]; then
       $CP /opt/stellar-default/$NETWORK/nginx/ $NXHOME
     fi
+    if thruput_installed; then
+      $CP /opt/stellar-default/thruput/nginx/ $NXHOME
+    fi
+  fi
+
+  if thruput_installed; then
+    if [ -d $THRUPUTHOME ]; then
+      echo "thruput: config directory exists, skipping copy"
+    else
+      $CP /opt/stellar-default/thruput/ $THRUPUTHOME
+    fi
   fi
 
   if [ -d $GALEXIEHOME/etc ]; then
@@ -658,6 +696,18 @@ function init_galexie() {
   popd
 }
 
+function thruput_installed() {
+  [ -x /usr/local/bin/thruput ]
+}
+
+function init_thruput() {
+  if [ "$ENABLE_THRUPUT" != "true" ]; then
+    return 0
+  fi
+
+  run_silent "chown-thruput" chown -R stellar:stellar $THRUPUTHOME
+}
+
 function kill_supervisor() {
   kill -3 $(cat "/var/run/supervisord.pid")
 }
@@ -896,6 +946,13 @@ function start_optional_services() {
     supervisorctl start ledger-meta-store
     supervisorctl start galexie
   fi
+
+  if [ "$ENABLE_THRUPUT" == "true" ]; then
+    supervisorctl start thruput-node
+    if [ "$ENABLE_HORIZON" == "true" ]; then
+      supervisorctl start thruput-accounts
+    fi
+  fi
 }
 
 function exec_supervisor() {
@@ -928,6 +985,9 @@ function print_service_logs() {
   if [ "$ENABLE_RPC" = "true" ]; then
     supervisorctl tail -f stellar-rpc  stdout > >(sed "s/^/${blue}stellar-rpc  | $clear/") &
   fi
+  if [ "$ENABLE_THRUPUT" = "true" ]; then
+    supervisorctl tail -f thruput-node stdout > >(sed "s/^/${yellow}thruput-node | $clear/") &
+  fi
 }
 
 # run_silent is a utility function that runs a command with an abbreviated
@@ -1015,6 +1075,9 @@ function service_status() {
     stellar_core_status "rpc" 11826 &
     stellar_rpc_status &
   fi
+  if [ "$ENABLE_THRUPUT" = "true" ]; then
+    thruput_status &
+  fi
 }
 
 function stellar_core_status() {
@@ -1048,6 +1111,19 @@ function stellar_rpc_status() {
   echo "stellar-rpc: up and ready"
 }
 
+function thruput_status() {
+  echo "thruput: waiting for ready state..."
+  while ! curl --silent --fail --request POST 'http://localhost:8000/thruput/rpc' \
+      --header 'Content-Type: application/json' \
+      --data-raw '{ "jsonrpc": "2.0", "id": 1, "method": "getChainTip" }' |
+      jq --exit-status '.result != null' > /dev/null 2>&1;
+  do
+    sleep 5
+  done
+  echo "thruput: up and ready"
+  echo "thruput: rpc: http://localhost:8000/thruput/rpc"
+}
+
 function horizon_status() {
   COUNTER=1
   echo "horizon: waiting for ingestion to catch up..."