diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 93c5fe2..517d051 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,6 +14,24 @@ jobs: - name: Validate hoprd.cfg.yaml.tpl run: ./test/validate-hoprd-config.sh + - name: Check that VPN clients can't reach local resources + run: ./test/firewall.sh + + - name: Check the blocklist and per-peer shaping + run: ./test/traffic.sh + + # Same checks inside the image that ships, so a missing bash/iproute2/iptables/ipset/curl + # fails here rather than on a DAppNode. Pulls the same base image the package build below + # does. + - name: Build the gnosisvpn-server image for the image checks + run: docker build -t gnosisvpn-server:checks gnosisvpn-server + + - name: Check the firewall inside the built gnosisvpn-server image + run: FIREWALL_TEST_IMAGE=gnosisvpn-server:checks ./test/firewall.sh + + - name: Check the blocklist and shaping inside the built gnosisvpn-server image + run: TRAFFIC_TEST_IMAGE=gnosisvpn-server:checks ./test/traffic.sh + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/Dockerfile b/Dockerfile index da87848..6818f1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG UPSTREAM_VERSION # hoprd-pix-test, not hoprd: the plain image enables no PIX pool feature, so it inherits # hopr-lib's default `pix-bjj` and announces BabyJubJub as its PIX curve suite. # Change back to `hoprd` once Curvy pool is ready -FROM europe-west3-docker.pkg.dev/hoprassociation/docker-images/hoprd-pix-test:${UPSTREAM_VERSION} +FROM europe-west3-docker.pkg.dev/hoprassociation/docker-images/hoprd:${UPSTREAM_VERSION} # not used at the moment, but might be useful in the future ENV DAPPNODE=true diff --git a/README.md b/README.md index d2f4359..02e69c3 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,114 @@ Runs a [GnosisVPN](https://vpn.gnosis.eth.limo) exit node on DAppNode: a HOPR no across installs, and isn't guaranteed stable across a `gnosisvpn-server` recreate - see "Known limitations" below. +## Firewall + +`gnosisvpn-server` has its own network namespace, and +[`gnosisvpn-server/wg-hooks.sh`](gnosisvpn-server/wg-hooks.sh) sets its forwarding policy. +`wg-quick` runs the script from `PostUp`/`PreDown` in `wggvpn.conf`. It is a port of the +`wg-hooks.sh` + `iptables.rules` pair from the exit-node appliance image (`gnosis_vpn-exit_os`). +The appliance version can't be used as-is, because a DAppNode package never sees the host's +network namespace (see "Known limitations" below). + +VPN clients are untrusted. Without the firewall, their traffic can reach the DAppNode host, the +dappmanager, every package on `dncore_network` and the LAN behind the host. With it, only +public destinations are reachable: + +- FORWARD policy is DROP. Client traffic goes through `GVPN_PRIVATE` first, which drops: + - RFC1918, CGNAT, link-local, loopback, `0.0.0.0/8` and multicast/reserved + - `dncore_network` (`172.33.0.0/16`), which is not in RFC1918 + - the subnet this container is attached to +- Only `wggvpn -> ` is accepted. Return traffic is `RELATED,ESTABLISHED` only, so the + exit node can't open connections to clients. +- INPUT from `wggvpn` accepts only the ICMP echo that the GnosisVPN client sends to check the + tunnel (`connection.ping`, default `10.128.0.1`). The control API on port 8000 and all other + local services are not reachable from the tunnel. +- `up` removes its old rules first, so a restart doesn't add duplicates. `down` leaves FORWARD + at DROP. +- `entrypoint.sh` sets FORWARD to DROP before it starts `gnosis_vpn-server`. `wg-quick` brings + `wggvpn` up before `PostUp` runs the hook, so this closes that gap. + +Not covered: + +- IPv6. `gnosis_vpn-server` adds peers with IPv4 `allowed-ips` only and `wggvpn` has no IPv6 + address, so WireGuard drops IPv6 client packets. The appliance also drops IPv6 with + `ip6tables`; this package does not. Re-check this when `gnosis_vpn-server` gets IPv6 support + ([gnosis/gnosis_vpn-client#330](https://github.com/gnosis/gnosis_vpn-client/issues/330)). +- The DAppNode's own public IPv4 address. It is a public destination, so it passes. (The + appliance finds and blocks it with an external HTTPS lookup; this package doesn't.) What a + client reaches there depends on where the address lives: + - On the host's NIC (a VPS, or a public IP from the ISP): every service the host listens on + and every published port, e.g. SSH and the hoprd API on `3001`. Don't run this package on + such a host unless the host firewall restricts those services by source. + - On a router in front of the host: whatever the router hairpins back, e.g. its admin UI if + it listens on WAN, and every forwarded port. Assume NAT loopback is on. +`just test-firewall` and `just test-firewall-image` check these rules (see +[`test/README.md`](test/README.md)). + +## Blocklist + +[`gnosisvpn-server/gvpn-blocklist-update.sh`](gnosisvpn-server/gvpn-blocklist-update.sh) fills +the `gvpn-blocklist` ipset, and `wg-hooks.sh` drops traffic to it, and traffic from it back to +clients, before the client ACCEPT. Ported from the appliance, which uses the same FireHOL level1 +feed: + +- `entrypoint.sh` loads the cached list before it starts the server, so the last known-good list + filters the first forwarded packet. The cache lives in the `gnosisvpn-data` volume. +- It then runs the refresher as a child process. A container has no timer, so the appliance's + twice-daily systemd timer becomes "refresh now, then every 12 h plus up to 1 h of jitter". +- A refresh swaps the whole set in atomically (`ipset swap`), so forwarding never sees a + half-built list. +- It fails open. An unreachable feed, or one that parses to fewer than 1000 or more than 131072 + entries, leaves the previous list in place and logs why. A poisoned `0.0.0.0/1` can't + blackhole every session, because prefixes shorter than /8 are only accepted for the 224/3 + bogon range. +- IPv4 only, like the isolation rules - see "Not covered" above for why client IPv6 never + reaches the tunnel. + +Feeds are configured from the DAppNode UI, see "Settings in the DAppNode UI" below. + +## Per-peer shaping + +[`gnosisvpn-server/gvpn-peer-shaping.sh`](gnosisvpn-server/gvpn-peer-shaping.sh) builds an HTB +tree on `wggvpn` when `wg-hooks.sh` runs, with one class per peer slot: + +- Download (server to client) is classified by destination address on `wggvpn` egress. +- Upload (client to internet) is mirrored from `wggvpn` ingress onto an `ifb-gvpn` device and + classified there by source address. It has to happen on the `wggvpn` side: the uplink only + ever sees post-MASQUERADE addresses. +- Each peer is capped at `GVPN_PEER_RATE_KBIT` (default 10000) and all peers together at + `GVPN_PEER_SHAPING_AGGREGATE_KBIT` (default 25000). It queues rather than drops, so TCP + settles at the cap instead of sawtoothing, and fq_codel signals with delay before loss + because a dropped packet costs an end-to-end retransmit across the mixnet. +- `entrypoint.sh` derives the peer range from `WIREGUARD_CIDR` and `MAX_SLOTS`, the same way it + derives the pool for `config.toml`, and exports it for the `wg-quick` process that runs the + hook. +- Unlike the appliance, a shaping failure does not stop the exit node. `tc` and the `ifb`/`sch_*` + modules come from the DAppNode host's kernel, and this container can't load modules into it. A + host without them serves unshaped and logs `per-peer shaping failed` on every interface sync. + +`just test-traffic` and `just test-traffic-image` check both of these (see +[`test/README.md`](test/README.md)). + +## Settings in the DAppNode UI + +The exit node's limits and blocklist are optional fields in the package's setup wizard, so they +are set in the DAppNode UI (the package's **Config**, or the wizard on install) rather than by +editing `docker-compose.yml`. Leaving a field empty keeps the default: + +| UI field | Variable | Default | +|---|---|---| +| Per-client bandwidth limit | `GVPN_PEER_RATE_KBIT` | `10000` kbit/s | +| Total bandwidth limit for all clients | `GVPN_PEER_SHAPING_AGGREGATE_KBIT` | `25000` kbit/s | +| Blocklist feeds | `GVPN_BLOCKLIST_FEEDS` | FireHOL level1 | +| Blocklist refresh interval | `GVPN_BLOCKLIST_REFRESH_INTERVAL_S` | `43200` s | +| WireGuard client subnet | `WIREGUARD_CIDR` | `10.128.0.0/24` | +| Maximum connected peers | `MAX_SLOTS` | `4` | + +Feed URLs are space-separated; one per line works too. The rest of the `GVPN_BLOCKLIST_*` +variables (minimum entries, download timeout, jitter) stay environment-only, with the +appliance's defaults. + ## Known limitations vs. production A regular (non-core) DAppNode package can't get `network_mode: host` or a custom docker diff --git a/docker-compose.yml b/docker-compose.yml index a74199f..6e71b19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: context: . args: # see https://console.cloud.google.com/artifacts/docker/hoprassociation/europe-west3/docker-images/hoprd - UPSTREAM_VERSION: 5.0.0-rc.1-commit.7d90171 + UPSTREAM_VERSION: 5.0.2-pr.232 ports: - "1422:1422/udp" - "1422:1422/tcp" @@ -65,6 +65,13 @@ services: WIREGUARD_CIDR: "10.128.0.0/24" WIREGUARD_MTU: "1420" MAX_SLOTS: "4" + # Per-peer traffic shaping, in kbit/s: each client is capped at the per-peer rate and + # all clients together at the aggregate. See README's "Per-peer shaping". + GVPN_PEER_RATE_KBIT: "10000" + GVPN_PEER_SHAPING_AGGREGATE_KBIT: "25000" + # Space-separated threat feeds for the blocklist. The last known-good list is cached + # in the gnosisvpn-data volume. See README's "Blocklist". + GVPN_BLOCKLIST_FEEDS: "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset" restart: unless-stopped deploy: resources: diff --git a/gnosisvpn-server/Dockerfile b/gnosisvpn-server/Dockerfile index b4858d7..b045234 100644 --- a/gnosisvpn-server/Dockerfile +++ b/gnosisvpn-server/Dockerfile @@ -1,17 +1,20 @@ FROM europe-west3-docker.pkg.dev/gnosisvpn-production/docker-images/gnosis_vpn-server:0.7.0@sha256:443c4264bb41a82526ea7d227676f5060633e489569b4fb7fe6299f6a28ff78d -# The base image (Alpine) ships wg/wg-quick/ip but not iptables/ipset, which the PostUp/PreDown -# rules in wggvpn.conf.tpl need (ported from the gnosisvpn-server Ansible role). -RUN apk add --no-cache iptables ip6tables ipset bash +# The scripts need bash, iproute2 (ip, tc), iptables and ipset, and the blocklist +# refresher needs curl. Install them explicitly instead of relying on the base image +# (Alpine); apk skips any that are already installed. +RUN apk add --no-cache bash iproute2 iptables ipset curl WORKDIR /app # Replace the image's own config.toml/wggvpn.conf with our templated versions. entrypoint.sh # renders these and execs gnosis_vpn-server directly (not via the image's own wrapper.sh, # which would leave it as a child process instead of PID 1 - see entrypoint.sh for why that -# matters for shutdown). +# matters for shutdown). PostUp/PreDown in wggvpn.conf run wg-hooks.sh, the forwarding +# policy - see README's "Firewall" section. wg-hooks.sh runs gvpn-peer-shaping.sh, and +# entrypoint.sh runs gvpn-blocklist-update.sh. COPY config.toml.tpl wggvpn.conf.tpl ./ -COPY entrypoint.sh ./entrypoint.sh -RUN chmod +x ./entrypoint.sh +COPY entrypoint.sh wg-hooks.sh gvpn-blocklist-update.sh gvpn-peer-shaping.sh ./ +RUN chmod +x ./entrypoint.sh ./wg-hooks.sh ./gvpn-blocklist-update.sh ./gvpn-peer-shaping.sh ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/gnosisvpn-server/entrypoint.sh b/gnosisvpn-server/entrypoint.sh index bb55ec5..bd80bbe 100755 --- a/gnosisvpn-server/entrypoint.sh +++ b/gnosisvpn-server/entrypoint.sh @@ -15,6 +15,19 @@ wg_address="${o1}.${o2}.${o3}.$((o4 + 1))" wg_start="${o1}.${o2}.${o3}.$((o4 + 2))" wg_end="${o1}.${o2}.${o3}.$((o4 + 1 + max_slots))" +# The pool has to stay inside the CIDR's last /24: per-peer shaping steers packets by +# the last octet only, and .255 is the broadcast address. +if (( o4 + 1 + max_slots > 254 )); then + echo "gnosisvpn-server: FATAL: WIREGUARD_CIDR ${WIREGUARD_CIDR} with MAX_SLOTS=${max_slots} needs last octets .$((o4 + 1))-.$((o4 + 1 + max_slots)), past .254" >&2 + exit 1 +fi + +# Consumed by wg-hooks.sh through gvpn-peer-shaping.sh, which wg-quick runs with this +# process's environment (gnosis_vpn-server spawns wg-quick as a plain child process). +export GVPN_PEER_SUBNET_PREFIX="${o1}.${o2}.${o3}" +export GVPN_PEER_MIN_OCTET=$((o4 + 2)) +export GVPN_PEER_MAX_OCTET=$((o4 + 1 + max_slots)) + # gnosis_vpn-server's own wrapper.sh generates a WireGuard key when PRIVATE_KEY isn't set, # but doesn't persist it - a fresh key on every restart would break peer registration. # Persist it in the /data volume instead, generating it once here. @@ -53,6 +66,18 @@ if [ "${current_ip_forward}" != "1" ]; then echo "gnosisvpn-server: WARNING: net.ipv4.ip_forward is '${current_ip_forward}', not '1', and this container cannot set it itself. VPN client traffic will not reach the internet until it's enabled on the DAppNode host." >&2 fi +# wg-quick brings wggvpn up (with its peers) before PostUp runs wg-hooks.sh, so close +# FORWARD now; wg-hooks.sh then opens only the public path. If this fails, the container +# fails: the exit node must not start without it. +iptables -P FORWARD DROP + +# The blocklist set lives in this container's namespace, so its cache lives in the /data +# volume and the refresh runs as a child process: a container has no timer. Loading the +# cache first (no network) filters the first forwarded packet with the last known-good +# list; the refresher then replaces it from the feeds in the background, failing open. +/app/gvpn-blocklist-update.sh --from-cache +/app/gvpn-blocklist-update.sh --loop & + # Run the binary directly as PID 1 (not via the image's wrapper.sh, which launches it as a # child process) so Docker's SIGTERM on `stop`/`down` reaches gnosis_vpn-server itself. It # has its own graceful-shutdown handling, which is what actually runs wg-quick down/PreDown. diff --git a/gnosisvpn-server/gvpn-blocklist-update.sh b/gnosisvpn-server/gvpn-blocklist-update.sh new file mode 100755 index 0000000..6119c2d --- /dev/null +++ b/gnosisvpn-server/gvpn-blocklist-update.sh @@ -0,0 +1,253 @@ +#!/bin/bash +# Populates the blocklist ipset the FORWARD DROP rules in wg-hooks.sh use. +# Ported from gvpn-exit-os (itself from gnosis_vpn-infrastructure, +# roles/gnosisvpn-blocklist): the Ansible variables arrive here as environment +# variables instead of exit.env. +# +# Three modes: +# --from-cache load the local cache file only (no network). entrypoint.sh +# runs this before gnosis_vpn-server, so the last known-good +# list is in effect for the first forwarded packet. +# --refresh fetch the threat feeds, refresh the cache, swap into the set. +# --loop refresh now, then every REFRESH_INTERVAL_S plus jitter. The +# appliance uses a systemd timer for this; a container has no +# timer, so entrypoint.sh runs this mode as a child process. +# +# Populating the live set always goes through an atomic `ipset swap`, so in-flight +# packets never see a partially-built set. The set is fail-open: any degraded +# path keeps the existing set (or an empty one) and the script still exits 0, +# reporting the problem in the log rather than failing the exit node. +set -euo pipefail + +IPSET_NAME="${GVPN_BLOCKLIST_IPSET_NAME:-gvpn-blocklist}" +IPSET_STAGING="${IPSET_NAME}-staging-$$" +IPSET_MAXELEM="${GVPN_BLOCKLIST_MAXELEM:-131072}" +CACHE_DIR="${GVPN_BLOCKLIST_CACHE_DIR:-/data/blocklist}" +CACHE_FILE="${CACHE_DIR}/blocklist.cidr" +MIN_ENTRIES="${GVPN_BLOCKLIST_MIN_ENTRIES:-1000}" +CURL_TIMEOUT="${GVPN_BLOCKLIST_CURL_TIMEOUT:-30}" +# Hard cap on each feed's download size. Bounds disk/memory use if a feed is +# compromised or misbehaving and streams an unexpectedly large body. +MAX_FILESIZE="${GVPN_BLOCKLIST_MAX_FILESIZE:-67108864}" +REFRESH_INTERVAL_S="${GVPN_BLOCKLIST_REFRESH_INTERVAL_S:-43200}" +REFRESH_JITTER_S="${GVPN_BLOCKLIST_REFRESH_JITTER_S:-3600}" +# Space-separated feed URLs. +FEEDS="${GVPN_BLOCKLIST_FEEDS:-https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset}" + +# Space- or newline-separated, so a value entered with one URL per line works +# too: `read` on its own would keep only the first line. +read -r -a FEED_LIST <<<"${FEEDS//$'\n'/ }" + +TMP_DIR="" +RESTORE_FILE="" + +usage() { + echo "usage: ${0} --from-cache|--refresh|--loop" >&2 + exit 64 +} + +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] gvpn-blocklist: $*"; } + +cleanup() { + [[ -n "${RESTORE_FILE}" ]] && rm -f "${RESTORE_FILE}" + [[ -n "${TMP_DIR}" ]] && rm -rf "${TMP_DIR}" + ipset destroy "${IPSET_STAGING}" 2>/dev/null || true +} +trap cleanup EXIT + +# Strip comments (; and #), drop whitespace, keep only valid IPv4 / IPv4-CIDR +# tokens (bare IPs are valid hash:net members, treated as /32). +# +# IPv4-only by design: the live set is `family inet` and this parser rejects any +# non-IPv4 token. That is safe only while client IPv6 never enters the tunnel, +# which holds because gnosis_vpn-server adds every peer with an IPv4 /32 +# allowed-ips and wggvpn carries no IPv6 address. If that ever changes, blocked +# destinations become reachable over IPv6 and a family inet6 set is needed. +# +# The octet (0-255) and prefix (/0../32) sub-patterns are validated here so that +# ipset never sees an out-of-range token that would abort the whole restore. +# Prefixes shorter than /8 are accepted only for the statically dropped 224/3 +# bogon range: a single 0.0.0.0/1 or 128.0.0.0/1 would send every VPN session +# into the blocklist DROP while the entry-count checks stay green. +parse_feed() { + sed -e 's/[;#].*$//' -e 's/[[:space:]]//g' \ + | grep -E '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(/(3[0-2]|[12][0-9]|[0-9]))?$' \ + | awk -F '[./]' 'NF == 4 || $5 >= 8 || $0 == "224.0.0.0/3"' || true +} + +# Whether a parsed entry count is within the acceptable [MIN_ENTRIES, IPSET_MAXELEM] +# range. A count outside this band is treated as a degraded/broken list and the +# caller keeps the existing set instead of swapping (fail-safe). +count_within_bounds() { + local count="$1" + if [[ "${count}" -lt "${MIN_ENTRIES}" ]]; then + log "ERROR ${count} entries (< ${MIN_ENTRIES}); keeping existing set" + return 1 + fi + if [[ "${count}" -gt "${IPSET_MAXELEM}" ]]; then + log "ERROR ${count} entries (> maxelem ${IPSET_MAXELEM}); keeping existing set" + return 1 + fi + return 0 +} + +# Ensure the live set exists so the iptables DROP rules always resolve, even when +# we have nothing to populate it with. +ensure_live_set() { + ipset create "${IPSET_NAME}" hash:net family inet maxelem "${IPSET_MAXELEM}" -exist +} + +# Build and validate the staging set from $1. This is intentionally separated +# from cache publication so malformed feed/cache data never becomes the new +# known-good cache. +stage_set() { + local src="$1" + ipset destroy "${IPSET_STAGING}" 2>/dev/null || true + RESTORE_FILE=$(mktemp) || return 1 + { + echo "create ${IPSET_STAGING} hash:net family inet maxelem ${IPSET_MAXELEM}" + sed "s|^|add ${IPSET_STAGING} |" "${src}" + } >"${RESTORE_FILE}" || return 1 + # -exist makes the adds idempotent: a duplicate CIDR (e.g. a bare IP from one + # feed colliding with the same X/32 from another) no longer aborts the whole + # restore. + ipset restore -exist <"${RESTORE_FILE}" || return 1 + rm -f "${RESTORE_FILE}" + RESTORE_FILE="" +} + +# Atomically replace the live set's contents with the already-validated CIDRs in +# $1. +swap_in() { + local src="$1" + stage_set "${src}" || return 1 + ensure_live_set || return 1 + ipset swap "${IPSET_STAGING}" "${IPSET_NAME}" || return 1 + ipset destroy "${IPSET_STAGING}" 2>/dev/null || true +} + +publish_cache() { + local src="$1" + mkdir -p "${CACHE_DIR}" || return 1 + cp "${src}" "${CACHE_FILE}.tmp" || return 1 + mv "${CACHE_FILE}.tmp" "${CACHE_FILE}" || return 1 +} + +# Entries currently in the live set, for the log. +live_entries() { + ipset list "${IPSET_NAME}" 2>/dev/null | awk '/Number of entries/ { print $4 }' +} + +# Log what the live set holds after a degraded run: the set is fail-open, so a +# failed refresh keeps the previous list and shows up here only. +report_live_set() { + local entries + entries="$(live_entries || true)" + log "live set holds ${entries:-unknown} entries" +} + +populate_from_cache() { + local count + if [[ ! -s "${CACHE_FILE}" ]]; then + log "ERROR cache file ${CACHE_FILE} missing or empty; leaving set empty" + ensure_live_set || log "ERROR could not create ${IPSET_NAME}" + report_live_set + return 0 + fi + count=$(wc -l <"${CACHE_FILE}") + if ! count_within_bounds "${count}"; then + report_live_set + return 0 + fi + if ! swap_in "${CACHE_FILE}"; then + log "ERROR cache failed ipset validation; keeping existing set" + report_live_set + return 0 + fi + log "populated from cache: ${count} entries" +} + +refresh() { + local all url feed_failed=0 feed_success=0 count + # --loop calls this repeatedly and never reaches the EXIT trap, so each run + # clears the previous run's directory. + [[ -n "${TMP_DIR}" ]] && rm -rf "${TMP_DIR}" + TMP_DIR=$(mktemp -d) + all="${TMP_DIR}/all.cidr" + : >"${all}" + for url in "${FEED_LIST[@]}"; do + if curl -fsSL --max-time "${CURL_TIMEOUT}" --max-filesize "${MAX_FILESIZE}" --retry 3 --retry-delay 5 "${url}" >"${TMP_DIR}/feed"; then + parse_feed <"${TMP_DIR}/feed" >>"${all}" + feed_success=$((feed_success + 1)) + log "fetched ${url}" + else + feed_failed=1 + log "WARN failed to fetch ${url}" + fi + done + + if [[ "${feed_success}" -eq 0 ]]; then + log "ERROR all feeds failed; falling back to cache" + populate_from_cache + return 0 + fi + if [[ "${feed_failed}" -ne 0 ]]; then + log "ERROR one or more feeds failed; keeping existing set and cache" + report_live_set + return 0 + fi + + sort -u "${all}" -o "${all}" + count=$(wc -l <"${all}") + if ! count_within_bounds "${count}"; then + report_live_set + return 0 + fi + + if ! swap_in "${all}"; then + log "ERROR parsed CIDR list failed ipset validation; keeping existing set and cache" + report_live_set + return 0 + fi + if ! publish_cache "${all}"; then + log "ERROR failed to update cache after live swap" + report_live_set + return 0 + fi + log "refresh complete: ${count} entries" +} + +loop() { + local delay interval="${REFRESH_INTERVAL_S}" jitter="${REFRESH_JITTER_S}" + # A value that is not a number makes `sleep` fail, which would turn this into + # a busy refresh loop against the feeds. Fall back to the defaults instead. + if ! [[ "${interval}" =~ ^[1-9][0-9]*$ ]]; then + log "WARN GVPN_BLOCKLIST_REFRESH_INTERVAL_S='${interval}' is not a positive integer; using 43200" + interval=43200 + fi + if ! [[ "${jitter}" =~ ^[0-9]+$ ]]; then + log "WARN GVPN_BLOCKLIST_REFRESH_JITTER_S='${jitter}' is not a whole number; using 3600" + jitter=3600 + fi + while :; do + refresh || true + delay=$((interval + RANDOM % (jitter + 1))) + log "next refresh in ${delay}s" + sleep "${delay}" + done +} + +main() { + case "${1:-}" in + --from-cache) populate_from_cache ;; + --refresh) refresh ;; + --loop) loop ;; + *) usage ;; + esac +} + +# Only dispatch when executed directly, not when sourced (the check scripts +# exercise parse_feed and count_within_bounds in isolation). +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/gnosisvpn-server/gvpn-peer-shaping.sh b/gnosisvpn-server/gvpn-peer-shaping.sh new file mode 100755 index 0000000..48739b2 --- /dev/null +++ b/gnosisvpn-server/gvpn-peer-shaping.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Per-peer traffic shaping for the exit node (`gvpn-peer-shaping.sh up|down`). +# Invoked by wg-hooks.sh from the wggvpn PostUp/PreDown hooks. Ported from +# gvpn-exit-os (itself from gnosis_vpn-infrastructure, roles/gnosisvpn-server): +# the Ansible variables arrive here as the environment entrypoint.sh exports. +# +# Shapes (queues) instead of policing (dropping): each peer gets an HTB class +# capped at the per-peer ceil with an fq_codel leaf, so TCP settles smoothly at +# the cap instead of sawtoothing on hard packet loss. The parent class enforces +# the node-wide aggregate cap. +# +# Directions: +# - download (server -> peers): tree on wggvpn egress, classified by dst IP +# - upload (peers -> internet): wggvpn ingress mirrored to an ifb device, tree +# on the ifb classified by src IP. Shaping has to happen on the wggvpn side +# because the uplink only ever sees post-MASQUERADE addresses. +# +# Peers are assigned GVPN_PEER_MIN_OCTET..GVPN_PEER_MAX_OCTET dynamically by +# gnosis_vpn-server, so all classes in that range are provisioned up front and +# packets are steered by a u32 hash table on the last octet (O(1) per packet). +# .1 is the server itself and .255 the broadcast address; both stay unshaped in +# the default class. The range must stay within one /24 (this script only +# varies the last octet), which entrypoint.sh enforces when it derives it. +set -euo pipefail + +WG_IF="wggvpn" +IFB_IF="ifb-gvpn" +# Derived by entrypoint.sh from WIREGUARD_CIDR and MAX_SLOTS, the same way it +# derives config.toml's pool and the rendered WireGuard address. Only `up` +# needs them, so `down` still works when they are absent. +SUBNET_PREFIX="${GVPN_PEER_SUBNET_PREFIX:-}" +PEER_MIN="${GVPN_PEER_MIN_OCTET:-}" +PEER_MAX="${GVPN_PEER_MAX_OCTET:-}" +PEER_RATE_KBIT="${GVPN_PEER_RATE_KBIT:-10000}" +AGGREGATE_KBIT="${GVPN_PEER_SHAPING_AGGREGATE_KBIT:-25000}" +# Guaranteed per-peer share; 253 x 56kbit stays below the aggregate so HTB +# guarantees remain honest. The ceil is what actually caps a peer. +PEER_GUARANTEE_KBIT=56 +# fq_codel tuned to signal congestion by queueing delay rather than drops: a +# dropped packet triggers an end-to-end retransmit across the mixnet, while a +# delayed packet lets the mixnet and TCP adapt to the slower pace. target +# tolerates ~250ms of standing queue (on top of the ~250ms mixnet peer RTT) +# before codel starts dropping, and ECN-capable flows are marked instead of +# dropped. Small quantum for inter-flow latency at low rates. +FQ_CODEL_OPTS="target 250ms interval 1000ms quantum 300 ecn" +# HTB derives a class's quantum from its rate and the parent's r2q, and warns on +# every class when the rates span too wide a range for one r2q (a 56kbit +# guarantee next to a 25Mbit aggregate does). Pin it at one MTU instead, so the +# container log isn't full of tc warnings on every interface sync. +HTB_QUANTUM=1500 + +usage() { + echo "usage: ${0} up|down" >&2 + exit 64 +} + +# Map a peer's last octet to the hex minor id of its class (1:). +peer_classid() { + printf '%x' "${1}" +} + +# IPv4 header offset of the address to classify on: dst for download (traffic +# leaving wggvpn toward peers), src for upload (peer traffic entering wggvpn, +# seen on the ifb mirror). +match_offset() { + case "${1}" in + dst) echo 16 ;; + src) echo 12 ;; + *) + echo "unknown match direction: ${1}" >&2 + return 1 + ;; + esac +} + +# Print the full tc command batch for one device. ${2} is the match direction +# (dst|src). Consumed via `tc -batch -` so the whole tree is built in one +# netlink session instead of ~800 tc invocations. +emit_tree() { + local dev="${1}" dir="${2}" offset octet minor + offset="$(match_offset "${dir}")" + + echo "qdisc add dev ${dev} root handle 1: htb default fff" + echo "class add dev ${dev} parent 1: classid 1:1 htb rate ${AGGREGATE_KBIT}kbit ceil ${AGGREGATE_KBIT}kbit quantum ${HTB_QUANTUM}" + # non-peer traffic (server address, broadcasts) shares the aggregate uncapped + echo "class add dev ${dev} parent 1:1 classid 1:fff htb rate 1000kbit ceil ${AGGREGATE_KBIT}kbit quantum ${HTB_QUANTUM}" + echo "qdisc add dev ${dev} parent 1:fff fq_codel ${FQ_CODEL_OPTS}" + echo "filter add dev ${dev} parent 1: prio 1 handle 2: protocol ip u32 divisor 256" + echo "filter add dev ${dev} parent 1: prio 1 protocol ip u32 ht 800:: match ip ${dir} ${SUBNET_PREFIX}.0/24 hashkey mask 0x000000ff at ${offset} link 2:" + + for octet in $(seq "${PEER_MIN}" "${PEER_MAX}"); do + minor="$(peer_classid "${octet}")" + echo "class add dev ${dev} parent 1:1 classid 1:${minor} htb rate ${PEER_GUARANTEE_KBIT}kbit ceil ${PEER_RATE_KBIT}kbit quantum ${HTB_QUANTUM}" + echo "qdisc add dev ${dev} parent 1:${minor} fq_codel ${FQ_CODEL_OPTS}" + echo "filter add dev ${dev} parent 1: prio 1 protocol ip u32 ht 2:${minor}: match ip ${dir} ${SUBNET_PREFIX}.${octet}/32 flowid 1:${minor}" + done +} + +# tc and the ifb/sch_* modules come from the DAppNode host's kernel, which this +# container cannot load modules into. A host without them gets an unshaped exit +# node and this error in the log, not a dead package. +up() { + # download: replace whatever root qdisc is present (idempotent re-runs) + tc qdisc del dev "${WG_IF}" root 2>/dev/null || true + emit_tree "${WG_IF}" dst | tc -batch - + + # upload: mirror wggvpn ingress onto the ifb and shape there + ip link add "${IFB_IF}" type ifb 2>/dev/null || true + ip link set "${IFB_IF}" up + tc qdisc del dev "${IFB_IF}" root 2>/dev/null || true + emit_tree "${IFB_IF}" src | tc -batch - + tc qdisc del dev "${WG_IF}" ingress 2>/dev/null || true + tc qdisc add dev "${WG_IF}" handle ffff: ingress + tc filter add dev "${WG_IF}" parent ffff: prio 1 protocol ip u32 match u32 0 0 action mirred egress redirect dev "${IFB_IF}" +} + +down() { + tc qdisc del dev "${WG_IF}" ingress 2>/dev/null || true + tc qdisc del dev "${WG_IF}" root 2>/dev/null || true + ip link del "${IFB_IF}" 2>/dev/null || true +} + +require_params() { + local missing=() + [[ -n "${SUBNET_PREFIX}" ]] || missing+=("GVPN_PEER_SUBNET_PREFIX") + [[ -n "${PEER_MIN}" ]] || missing+=("GVPN_PEER_MIN_OCTET") + [[ -n "${PEER_MAX}" ]] || missing+=("GVPN_PEER_MAX_OCTET") + if [[ "${#missing[@]}" -ne 0 ]]; then + echo "not set: ${missing[*]} (derived by entrypoint.sh)" >&2 + return 1 + fi + local value + for value in "${PEER_RATE_KBIT}" "${AGGREGATE_KBIT}"; do + if ! [[ "${value}" =~ ^[1-9][0-9]*$ ]]; then + echo "shaping rates must be positive integers in kbit/s, got '${value}'" >&2 + return 1 + fi + done +} + +main() { + case "${1:-}" in + up) + require_params || exit 64 + up + ;; + down) down ;; + *) usage ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/gnosisvpn-server/wg-hooks.sh b/gnosisvpn-server/wg-hooks.sh new file mode 100755 index 0000000..f8ed8c9 --- /dev/null +++ b/gnosisvpn-server/wg-hooks.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# wg-quick PostUp/PreDown hook for wggvpn (`wg-hooks.sh up|down`). +# +# Sets the exit node's forwarding policy in this container's network namespace, +# where gnosis_vpn-server creates the WireGuard interface. VPN clients are +# untrusted, so they may only reach public destinations: not the DAppNode host, +# not other packages on dncore_network, not the LAN behind the host. +# +# Rule order is the guarantee: the private-range jump is first in FORWARD, the +# client ACCEPT is last, and the policy is DROP. Ported from wg-hooks.sh + +# iptables.rules in the gvpn-exit-os appliance image, which a DAppNode package +# can't reuse as-is because it never sees the host's network namespace (see +# README). +set -euo pipefail + +WG_IF="wggvpn" +PRIVATE_CHAIN="GVPN_PRIVATE" +# Filled by gvpn-blocklist-update.sh, which entrypoint.sh starts; this hook only +# drops traffic against it, and leaves the set in the kernel on `down` so the +# refresher keeps working across interface restarts. +BLOCKLIST_SET="${GVPN_BLOCKLIST_IPSET_NAME:-gvpn-blocklist}" +BLOCKLIST_MAXELEM="${GVPN_BLOCKLIST_MAXELEM:-131072}" +SHAPING_BIN="${GVPN_SHAPING_BIN:-/app/gvpn-peer-shaping.sh}" + +usage() { + echo "usage: ${0} up|down [uplink]" >&2 + exit 64 +} + +[[ $# -ge 3 ]] || usage +action="$1" +client_cidr="$2" +server_address="$3" +# The container's DAppNode-managed interface; the host NATs it the rest of the +# way out. +uplink="${4:-eth0}" + +# Destinations a VPN client must never reach: RFC1918 (the LAN and Docker's +# default network pool), CGNAT, link-local, loopback, "this network" and +# multicast/reserved. +PRIVATE_NETS=( + 10.0.0.0/8 + 172.16.0.0/12 + 192.168.0.0/16 + 100.64.0.0/10 + 169.254.0.0/16 + 127.0.0.0/8 + 0.0.0.0/8 + 224.0.0.0/3 + # DAppNode's dncore_network (every package and the dappmanager). It is not in + # RFC1918, so without this entry the dappmanager stays reachable. + 172.33.0.0/16 +) + +PRIVATE_JUMP=(FORWARD -i "${WG_IF}" -j "${PRIVATE_CHAIN}") +BLOCKLIST_DST_DROP=(FORWARD -i "${WG_IF}" -m set --match-set "${BLOCKLIST_SET}" dst -j DROP) +BLOCKLIST_SRC_DROP=(FORWARD -o "${WG_IF}" -m set --match-set "${BLOCKLIST_SET}" src -j DROP) +CLIENT_ACCEPT=(FORWARD -i "${WG_IF}" -o "${uplink}" -j ACCEPT) +# shellcheck disable=SC2054 # the comma belongs to --ctstate, not the array +RETURN_ACCEPT=(FORWARD -o "${WG_IF}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT) +# The GnosisVPN client pings the WireGuard server address (`connection.ping`, +# default 10.128.0.1) to verify the tunnel. Accept only that echo request; +# everything else from clients is dropped, the control API on port 8000 included. +PING_ACCEPT=(INPUT -s "${client_cidr}" -d "${server_address}/32" -i "${WG_IF}" -p icmp -m icmp --icmp-type 8 -j ACCEPT) +CLIENT_DROP=(INPUT -i "${WG_IF}" -j DROP) +NAT_RULE=(POSTROUTING -s "${client_cidr}" -o "${uplink}" -j MASQUERADE) + +# The subnet this container is attached to, whatever DAppNode assigned, so +# isolation doesn't rely on the static entry above. Read from the kernel's +# route for the interface. +own_network() { + local network + # awk reads all input, so ip never hits a closed pipe and pipefail only + # reports a real ip failure. + if ! network="$(ip -4 route show dev "${uplink}" proto kernel scope link | awk 'NR == 1 { print $1 }')"; then + echo "wg-hooks: cannot read the routes of ${uplink}" >&2 + return 1 + fi + if [[ -z "${network}" ]]; then + echo "wg-hooks: no IPv4 subnet on ${uplink}; only the static ranges are dropped" >&2 + return 0 + fi + printf '%s\n' "${network}" +} + +private_nets() { + local own + own="$(own_network)" || return 1 + printf '%s\n' "${PRIVATE_NETS[@]}" "${own}" | awk 'NF && !seen[$0]++' +} + +up() { + # gnosis_vpn-server can bring the interface up again without PreDown (hard + # kill, interface sync), so clear old rules first to avoid duplicates. + down + + # Rules for a missing interface match nothing and raise no error, which + # would silently leave the control API reachable. Refuse instead. + if ! ip link show dev "${WG_IF}" >/dev/null 2>&1; then + echo "wg-hooks: ${WG_IF} does not exist; refusing to install forwarding rules" >&2 + exit 1 + fi + + # Fail closed first: a client packet arriving mid-install is dropped. + iptables -P FORWARD DROP + iptables -N "${PRIVATE_CHAIN}" 2>/dev/null || iptables -F "${PRIVATE_CHAIN}" + # Collected first, not read from a process substitution (bash ignores its + # failure): a range that can't be determined must stop the hook. + local nets net + nets="$(private_nets)" + while IFS= read -r net; do + iptables -A "${PRIVATE_CHAIN}" -d "${net}" -j DROP + done <<<"${nets}" + # The blocklist rules resolve the set name, so it has to exist first. The + # refresher fills this same set in place with ipset swap. + ipset create "${BLOCKLIST_SET}" hash:net family inet maxelem "${BLOCKLIST_MAXELEM}" -exist + + # -I without a rule number inserts at the head, ahead of anything else, so + # every drop lands in front of the client ACCEPT below. + iptables -I "${BLOCKLIST_DST_DROP[@]}" + iptables -I "${BLOCKLIST_SRC_DROP[@]}" + iptables -I "${PRIVATE_JUMP[@]}" + iptables -A "${CLIENT_ACCEPT[@]}" + iptables -A "${RETURN_ACCEPT[@]}" + iptables -I "${CLIENT_DROP[@]}" + iptables -I "${PING_ACCEPT[@]}" + + # tc and the ifb/sch_* modules come from the DAppNode host's kernel, which this + # container cannot load modules into. A host without them gets an unshaped exit + # node and this line in the log, rather than no exit node at all. + if ! "${SHAPING_BIN}" up; then + echo "wg-hooks: per-peer shaping failed; VPN clients are not rate limited" >&2 + fi + + iptables -t nat -A "${NAT_RULE[@]}" +} + +down() { + iptables -t nat -D "${NAT_RULE[@]}" 2>/dev/null || true + "${SHAPING_BIN}" down 2>/dev/null || true + iptables -D "${PING_ACCEPT[@]}" 2>/dev/null || true + iptables -D "${CLIENT_DROP[@]}" 2>/dev/null || true + iptables -D "${RETURN_ACCEPT[@]}" 2>/dev/null || true + iptables -D "${CLIENT_ACCEPT[@]}" 2>/dev/null || true + iptables -D "${PRIVATE_JUMP[@]}" 2>/dev/null || true + iptables -D "${BLOCKLIST_SRC_DROP[@]}" 2>/dev/null || true + iptables -D "${BLOCKLIST_DST_DROP[@]}" 2>/dev/null || true + iptables -F "${PRIVATE_CHAIN}" 2>/dev/null || true + iptables -X "${PRIVATE_CHAIN}" 2>/dev/null || true + # FORWARD stays DROP on purpose: nothing forwards without the interface, and + # a hook that failed half way must not leave the namespace open. + # The blocklist set stays too: emptying it on every interface sync would + # leave the blocklist inactive until the next refresh. +} + +main() { + case "${action}" in + up) up ;; + down) down ;; + *) usage ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/gnosisvpn-server/wggvpn.conf.tpl b/gnosisvpn-server/wggvpn.conf.tpl index 07e5979..0232e80 100644 --- a/gnosisvpn-server/wggvpn.conf.tpl +++ b/gnosisvpn-server/wggvpn.conf.tpl @@ -1,17 +1,16 @@ -# Rendered by entrypoint.sh from this template. PostUp/PreDown do the NAT a WireGuard exit -# node needs, scoped down to what's possible for a regular (non-core) DAppNode package: no -# network_mode: host, so this container's own network namespace (on dncore_network, like any -# other DAppNode package service - see docker-compose.yml) never reaches the DAppNode host's. -# MASQUERADE targets "eth0" (this container's own interface on its DAppNode-managed network) -# rather than a real public NIC; Docker's own host-level NAT then carries that traffic the -# rest of the way out, same as it does for any other container's egress traffic. Explicit -# DOCKER-USER firewall exceptions aren't included for the same reason (no host namespace -# access) - see README's "Known limitations vs. production" section for why that's not -# actually a gap here. +# Rendered by entrypoint.sh from this template. PostUp/PreDown run wg-hooks.sh, +# which installs the exit node's forwarding policy in this container's own +# network namespace. A regular DAppNode package gets no network_mode: host, so +# it never reaches the host's namespace (see docker-compose.yml). +# VPN clients may only reach public destinations. MASQUERADE targets "eth0" +# (this container's DAppNode-managed interface), and Docker's host-level NAT +# carries the traffic the rest of the way out. There are no DOCKER-USER +# exceptions for the same reason - see README's "Known limitations vs. +# production" for why that's not a gap here. [Interface] Address = __WG_ADDRESS__/32 ListenPort = 51820 PrivateKey = MTU = __WG_MTU__ -PostUp = iptables -t nat -A POSTROUTING -s __WG_CIDR__ -o eth0 -j MASQUERADE -PreDown = iptables -t nat -D POSTROUTING -s __WG_CIDR__ -o eth0 -j MASQUERADE +PostUp = /app/wg-hooks.sh up __WG_CIDR__ __WG_ADDRESS__ +PreDown = /app/wg-hooks.sh down __WG_CIDR__ __WG_ADDRESS__ diff --git a/justfile b/justfile index c39e2f6..edf2369 100644 --- a/justfile +++ b/justfile @@ -17,3 +17,21 @@ local-logs: # Re-validate hoprd.cfg.yaml against the real hoprd 4.0.3 config parser validate-config: ./test/validate-hoprd-config.sh + +# Check that VPN clients can't reach local resources through the exit node +test-firewall: + ./test/firewall.sh + +# Run the same firewall checks inside the built gnosisvpn-server image +test-firewall-image: + docker build -t gnosisvpn-server:firewall-test gnosisvpn-server + FIREWALL_TEST_IMAGE=gnosisvpn-server:firewall-test ./test/firewall.sh + +# Check the blocklist and per-peer shaping the exit node applies +test-traffic: + ./test/traffic.sh + +# Run the same blocklist and shaping checks inside the built image +test-traffic-image: + docker build -t gnosisvpn-server:traffic-test gnosisvpn-server + TRAFFIC_TEST_IMAGE=gnosisvpn-server:traffic-test ./test/traffic.sh diff --git a/setup-wizard.yml b/setup-wizard.yml index d3588d6..3839158 100644 --- a/setup-wizard.yml +++ b/setup-wizard.yml @@ -146,3 +146,64 @@ fields: secret: false pattern: ^([1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-2])$ patternErrorMessage: Must be a number between 1 and 252 + + - id: GVPN_PEER_RATE_KBIT + target: + type: environment + name: GVPN_PEER_RATE_KBIT + service: gnosisvpn-server + title: Per-client bandwidth limit (advanced) + required: false + description: |- + Each connected VPN client is limited to this rate in each direction, in kbit/s. Shaping + queues packets instead of dropping them, so TCP slows down smoothly at the limit. Leave + empty to use the default (`10000`, i.e. 10 Mbit/s). + secret: false + pattern: ^[1-9][0-9]*$ + patternErrorMessage: Must be a positive whole number of kbit/s, e.g. 10000 + + - id: GVPN_PEER_SHAPING_AGGREGATE_KBIT + target: + type: environment + name: GVPN_PEER_SHAPING_AGGREGATE_KBIT + service: gnosisvpn-server + title: Total bandwidth limit for all clients (advanced) + required: false + description: |- + All connected clients together are limited to this rate, in kbit/s. Keep it below your + uplink speed so the exit node cannot saturate your connection. Leave empty to use the + default (`25000`, i.e. 25 Mbit/s). + secret: false + pattern: ^[1-9][0-9]*$ + patternErrorMessage: Must be a positive whole number of kbit/s, e.g. 25000 + + - id: GVPN_BLOCKLIST_FEEDS + target: + type: environment + name: GVPN_BLOCKLIST_FEEDS + service: gnosisvpn-server + title: Blocklist feeds (advanced) + required: false + description: |- + Space-separated URLs of IPv4 threat feeds, each listing one address or CIDR per line. + Traffic from VPN clients to these addresses, and from them back to clients, is dropped. + Leave empty to use the default: FireHOL level1, the same feed GnosisVPN's own exit nodes + use. + secret: false + pattern: ^(https?://[^ ]+)( (https?://[^ ]+))*$ + patternErrorMessage: Must be one or more space-separated http(s) URLs + + - id: GVPN_BLOCKLIST_REFRESH_INTERVAL_S + target: + type: environment + name: GVPN_BLOCKLIST_REFRESH_INTERVAL_S + service: gnosisvpn-server + title: Blocklist refresh interval (advanced) + required: false + description: |- + How often the blocklist feeds are fetched, in seconds, plus up to one hour of jitter. + Leave empty to use the default (`43200`, twice a day). A refresh that fails, or that + parses to an implausible number of entries, keeps the previous list. + secret: false + pattern: ^[1-9][0-9]*$ + patternErrorMessage: Must be a positive whole number of seconds, e.g. 43200 diff --git a/test/README.md b/test/README.md index c482d99..f94fc3f 100644 --- a/test/README.md +++ b/test/README.md @@ -43,6 +43,54 @@ and renders it into `hoprd.cfg.yaml.tpl` - see that file and `docker-compose.yml resolve, so `docker-compose.local.yml` overrides `GNOSISVPN_SERVER_HOST` to plain `gnosisvpn-server`, which plain `docker compose` resolves via the project's default network. +## Checking the firewall + +```sh +just test-firewall # or: test/firewall.sh +just test-firewall-image # or: FIREWALL_TEST_IMAGE= test/firewall.sh +``` + +Checks that a VPN client can't reach local resources, but can still reach public destinations +and ping the tunnel-check address (see the README's "Firewall" section). Both commands: + +1. Grep `wggvpn.conf.tpl`, the `gnosisvpn-server` Dockerfile and `entrypoint.sh` for the + `wg-hooks.sh` wiring and the early FORWARD DROP. +2. Run `test/firewall-check.sh` in a throwaway privileged container. A veth pair named `wggvpn` + stands in for WireGuard, a dummy `eth1` for the uplink, and a separate network namespace for + the client, so the probes really go through the rules. Only Docker is needed. + +`test-firewall` uses `alpine` with the repo's `wg-hooks.sh` mounted in. `test-firewall-image` +uses the built `gnosisvpn-server` image with nothing mounted or installed, which proves the +image ships the hook and its tools (`bash`, `iproute2`, `iptables`). Building that image needs +access to GnosisVPN's registry: `gcloud auth configure-docker europe-west3-docker.pkg.dev`. + +## Checking the blocklist and shaping + +```sh +just test-traffic # or: test/traffic.sh +just test-traffic-image # or: TRAFFIC_TEST_IMAGE= test/traffic.sh +``` + +Checks the two things the exit node does to forwarded traffic besides isolating it (see the +README's "Blocklist" and "Per-peer shaping" sections). Both commands: + +1. Grep the `gnosisvpn-server` Dockerfile, `entrypoint.sh` and `wg-hooks.sh` for the wiring: the + image ships both scripts, the entrypoint loads the cached list and starts the refresher, and + the hook runs the shaping. +2. Run `test/traffic-check.sh` in a throwaway privileged container. On top of the firewall + suite's veth and client namespace there is a third namespace standing in for a public host + that answers pings, so "the client still reaches the internet" is a real reply rather than a + rule counter. Feeds are `file://` URLs, so no network is needed. + +It covers the blocklist parser and the fail-open paths (a feed below the minimum entry count, an +unreachable feed, reloading from the cache), that a blocklisted destination really is dropped +while a public one is not, and that every peer slot has an HTB class in both directions with +packets actually counted in it. + +`test-traffic` uses `alpine` with the repo's scripts mounted in. `test-traffic-image` uses the +built image with nothing mounted or installed, which proves it ships `ipset` and `curl` as well +as the scripts. + ## Validating hoprd.cfg.yaml.tpl on its own ```sh diff --git a/test/firewall-check.sh b/test/firewall-check.sh new file mode 100755 index 0000000..26fdd24 --- /dev/null +++ b/test/firewall-check.sh @@ -0,0 +1,359 @@ +#!/bin/bash +# Runs inside a throwaway privileged container (see test/firewall.sh), so the +# rules wg-hooks.sh installs land in a network namespace the test can discard. +# A veth pair named wggvpn stands in for WireGuard and a second namespace for a +# VPN client, so probes are really forwarded across wggvpn, not just checked +# against the ruleset. +set -euo pipefail + +HOOK=/app/wg-hooks.sh +WG_IF=wggvpn +CHAIN_PRIVATE=GVPN_PRIVATE +# The hook also runs the per-peer shaping; that is test/traffic-check.sh's +# subject, so stub it out here rather than mount it and depend on tc and ifb. +export GVPN_SHAPING_BIN=/bin/true +CLIENT_CIDR=10.128.0.0/24 +SERVER_ADDRESS=10.128.0.1 +CLIENT_ADDRESS=10.128.0.2 +# The uplink is a dummy on a public-looking range, so the subnet wg-hooks.sh +# derives from it can't be confused with a static private range. +UPLINK=eth1 +UPLINK_SUBNET=198.51.100.0/24 +# TEST-NET-3, routed out the uplink: public but never reachable, so a probe to +# it can only prove that FORWARD accepted it. +PUBLIC_PROBE=203.0.113.1 +CLIENT_NS=vpn-client +CLIENT_IF=veth-client + +# Every destination the hook should drop, including the uplink subnet it +# derives at runtime. +DROPPED_NETS=( + 10.0.0.0/8 + 172.16.0.0/12 + 192.168.0.0/16 + 100.64.0.0/10 + 169.254.0.0/16 + 127.0.0.0/8 + 0.0.0.0/8 + 224.0.0.0/3 + 172.33.0.0/16 + "${UPLINK_SUBNET}" +) + +failures=0 +pass() { echo "ok: $*"; } +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +setup() { + # The uplink is the default route, like eth0 on the real exit node, so a + # private destination the hook missed would leave through it and match the + # client ACCEPT. + ip link add "${UPLINK}" type dummy + ip addr add 198.51.100.10/24 dev "${UPLINK}" + ip link set "${UPLINK}" up + ip route replace default dev "${UPLINK}" + + # The tunnel, and the client behind it. + ip netns add "${CLIENT_NS}" + ip link add "${WG_IF}" type veth peer name "${CLIENT_IF}" + ip link set "${CLIENT_IF}" netns "${CLIENT_NS}" + ip addr add "${SERVER_ADDRESS}/24" dev "${WG_IF}" + ip link set "${WG_IF}" up + ip -n "${CLIENT_NS}" addr add "${CLIENT_ADDRESS}/24" dev "${CLIENT_IF}" + ip -n "${CLIENT_NS}" link set "${CLIENT_IF}" up + ip -n "${CLIENT_NS}" link set lo up + # Full tunnel: all client traffic goes through wggvpn (the client's + # wireguard.allowed_ips defaults to 0.0.0.0/0). + ip -n "${CLIENT_NS}" route add default via "${SERVER_ADDRESS}" + + # An address outside the uplink and tunnel networks, standing in for the + # exit node's other interfaces. + OTHER_ADDRESS="$(ip -4 -o addr show dev eth0 | awk '{ print $4 }' | cut -d/ -f1)" + [ -n "${OTHER_ADDRESS}" ] || { + echo "no IPv4 address on eth0 to probe the control API on" >&2 + exit 1 + } +} + +# Packet count of the rule in CHAIN whose `iptables -S -v` line contains SPEC, +# with the `-c PACKETS BYTES` counters stripped. Counters are zeroed before +# every probe, so the count belongs to that probe. +counter() { + local chain="$1" spec="$2" + iptables -t filter -S "${chain}" -v | awk -v want="${spec}" -v prefix="-A ${chain} " ' + { + line = $0 + pkts = "missing" + if (match(line, / -c [0-9]+ [0-9]+/)) { + split(substr(line, RSTART + 4, RLENGTH - 4), counts, " ") + pkts = counts[1] + line = substr(line, 1, RSTART - 1) substr(line, RSTART + RLENGTH) + } + # This chain only, so a spec cannot match a rule somewhere else. + if (index(line, prefix) == 1 && index(line, want) > 0) { print pkts; found = 1; exit } + } + END { if (!found) print "missing" } + ' +} + +# expect_rule_count CHAIN SPEC COUNT DESCRIPTION +expect_rule_count() { + local chain="$1" spec="$2" count="$3" description="$4" actual + actual="$(iptables -t filter -S "${chain}" | grep -cF -- "${spec}" || true)" + if [ "${actual}" -eq "${count}" ]; then + pass "${description}" + else + fail "${description}: ${actual} rules match '${spec}' in ${chain}, expected ${count}" + fi +} + +expect_matched() { + local chain="$1" spec="$2" description="$3" pkts + pkts="$(counter "${chain}" "${spec}")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule '${spec}' in ${chain}" + elif [ "${pkts}" -lt 1 ]; then + fail "${description}: rule '${spec}' in ${chain} matched ${pkts} packets" + else + pass "${description}" + fi +} + +expect_unmatched() { + local chain="$1" spec="$2" description="$3" pkts + pkts="$(counter "${chain}" "${spec}")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule '${spec}' in ${chain}" + elif [ "${pkts}" -ne 0 ]; then + fail "${description}: rule '${spec}' in ${chain} matched ${pkts} packets" + else + pass "${description}" + fi +} + +# expect_no_private_drop DESCRIPTION: none of the private-range rules matched. +expect_no_private_drop() { + local description="$1" net pkts + for net in "${DROPPED_NETS[@]}"; do + pkts="$(counter "${CHAIN_PRIVATE}" "-d ${net} -j DROP")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule for ${net} in ${CHAIN_PRIVATE}" + return 0 + elif [ "${pkts}" -ne 0 ]; then + fail "${description}: ${CHAIN_PRIVATE} dropped ${pkts} packets to ${net}" + return 0 + fi + done + pass "${description}" +} + +# probe_from_client DESTINATION: one ICMP echo sent as the VPN client. +probe_from_client() { + ip netns exec "${CLIENT_NS}" ping -c 1 -W 1 "$1" >/dev/null 2>&1 || true +} + +zero_counters() { + local chain + for chain in FORWARD INPUT "${CHAIN_PRIVATE}"; do + iptables -t filter -Z "${chain}" + done +} + +setup + +# --- the hook installs isolation, and installing it twice does not stack ------ +"${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" +"${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +forward_policy="$(iptables -t filter -S FORWARD | sed -n '1p')" +if [ "${forward_policy}" = "-P FORWARD DROP" ]; then + pass "FORWARD policy is DROP" +else + fail "FORWARD policy is '${forward_policy}', expected '-P FORWARD DROP'" +fi + +# hoprd opens new connections to the control API and WireGuard port from +# dncore_network via eth0, so INPUT must stay ACCEPT or the exit node breaks. +# Only the tunnel is filtered. +input_policy="$(iptables -t filter -S INPUT | sed -n '1p')" +if [ "${input_policy}" = "-P INPUT ACCEPT" ]; then + pass "INPUT policy is ACCEPT, so hoprd can still reach the exit node" +else + fail "INPUT policy is '${input_policy}', expected '-P INPUT ACCEPT'" +fi + +# One of each, not a stack of them from the second `up`. +expect_rule_count FORWARD "-i ${WG_IF}" 3 "FORWARD has one private jump, one blocklist drop and one client accept" +expect_rule_count INPUT "-i ${WG_IF}" 2 "INPUT has exactly one ping accept and one client drop" + +first_forward_rule="$(iptables -t filter -S FORWARD | sed -n '2p')" +if [ "${first_forward_rule}" = "-A FORWARD -i ${WG_IF} -j ${CHAIN_PRIVATE}" ]; then + pass "the private-range jump is the first FORWARD rule" +else + fail "first FORWARD rule is '${first_forward_rule}', expected the ${CHAIN_PRIVATE} jump" +fi + +for net in "${DROPPED_NETS[@]}"; do + if iptables -t filter -S "${CHAIN_PRIVATE}" | grep -qxF -- "-A ${CHAIN_PRIVATE} -d ${net} -j DROP"; then + pass "${CHAIN_PRIVATE} drops ${net}" + else + fail "${CHAIN_PRIVATE} does not drop ${net}" + fi +done + +masquerade_count="$(iptables -t nat -S POSTROUTING | grep -cF -- "-s ${CLIENT_CIDR} -o ${UPLINK} -j MASQUERADE" || true)" +if [ "${masquerade_count}" -eq 1 ]; then + pass "exactly one MASQUERADE rule for ${CLIENT_CIDR}" +else + fail "found ${masquerade_count} MASQUERADE rules for ${CLIENT_CIDR}, expected 1" +fi + +# --- a VPN client cannot reach private destinations --------------------------- +zero_counters +probe_from_client 192.168.1.1 +expect_matched "${CHAIN_PRIVATE}" "-d 192.168.0.0/16" "client to a LAN address hits the private drop" +expect_unmatched FORWARD "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" "client to a LAN address is not forwarded" + +zero_counters +probe_from_client 172.33.0.1 +expect_matched "${CHAIN_PRIVATE}" "-d 172.33.0.0/16" "client to the DAppNode host hits the private drop" +expect_unmatched FORWARD "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" "client to the DAppNode host is not forwarded" + +zero_counters +probe_from_client 198.51.100.20 +expect_matched "${CHAIN_PRIVATE}" "-d ${UPLINK_SUBNET}" "client to another host on the exit's own network hits the private drop" + +zero_counters +probe_from_client 10.99.0.1 +expect_matched "${CHAIN_PRIVATE}" "-d 10.0.0.0/8" "client to an address in the tunnel's own /8 hits the private drop" + +# --- a VPN client still reaches public destinations --------------------------- +zero_counters +probe_from_client "${PUBLIC_PROBE}" +expect_matched FORWARD "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" "client to a public address is forwarded" +expect_no_private_drop "client to a public address is not dropped by the private ranges" + +# --- the client's tunnel-verification ping still reaches the exit ------------- +zero_counters +probe_from_client "${SERVER_ADDRESS}" +expect_matched INPUT "-i ${WG_IF} -p icmp -m icmp --icmp-type 8 -j ACCEPT" "client ping to the WireGuard address is accepted" +expect_unmatched INPUT "-i ${WG_IF} -j DROP" "client ping to the WireGuard address is not dropped" + +# --- ...and nothing else from the tunnel reaches the exit --------------------- +# The drop is protocol- and port-agnostic, so an echo to the exit's own +# address proves the control API on it is unreachable. +zero_counters +probe_from_client "${OTHER_ADDRESS}" +expect_matched INPUT "-i ${WG_IF} -j DROP" "client to the exit's other addresses is dropped" + +# --- down removes everything the hook installed ------------------------------- +"${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +if iptables -t filter -S "${CHAIN_PRIVATE}" >/dev/null 2>&1; then + fail "${CHAIN_PRIVATE} still exists after down" +else + pass "${CHAIN_PRIVATE} is gone after down" +fi +if iptables -t filter -S FORWARD | grep -qF -- "-i ${WG_IF}"; then + fail "FORWARD still has ${WG_IF} rules after down" +else + pass "FORWARD has no ${WG_IF} rules after down" +fi +forward_policy_after_down="$(iptables -t filter -S FORWARD | sed -n '1p')" +if [ "${forward_policy_after_down}" = "-P FORWARD DROP" ]; then + pass "FORWARD stays DROP after down" +else + fail "FORWARD policy is '${forward_policy_after_down}' after down, expected DROP" +fi +if iptables -t nat -S POSTROUTING | grep -qF -- "-j MASQUERADE"; then + fail "POSTROUTING still masquerades after down" +else + pass "POSTROUTING has no MASQUERADE rule after down" +fi + +# --- a hook that fails half way leaves the namespace closed ------------------- +# Fail each install step in turn with a mocked iptables on PATH, then check the +# half-installed state still isolates clients: policy DROP, and the private +# jump before any ACCEPT. +closed_after_partial_install() { + local policy first + policy="$(iptables -t filter -S FORWARD | sed -n '1p')" + [ "${policy}" = "-P FORWARD DROP" ] || return 1 + first="$(iptables -t filter -S FORWARD | sed -n '2p')" + if iptables -t filter -S FORWARD | grep -q -- "-j ACCEPT"; then + [ "${first}" = "-A FORWARD -i ${WG_IF} -j ${CHAIN_PRIVATE}" ] || return 1 + fi + if iptables -t filter -S INPUT | grep -q -- "-i ${WG_IF} .*-j ACCEPT"; then + iptables -t filter -S INPUT | grep -qxF -- "-A INPUT -i ${WG_IF} -j DROP" || return 1 + fi + return 0 +} + +mock_bin="$(mktemp -d)" +real_iptables="$(command -v iptables)" +cat >"${mock_bin}/iptables" <&2 + exit 1 +fi +exec "${real_iptables}" "\$@" +MOCK +chmod +x "${mock_bin}/iptables" + +for step in "-j ACCEPT" "--ctstate" "--icmp-type"; do + "${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + if FAIL_ON="${step}" PATH="${mock_bin}:${PATH}" \ + "${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" 2>/dev/null; then + fail "the hook reported success although installing '${step}' failed" + else + pass "the hook fails when installing '${step}' fails" + fi + if closed_after_partial_install; then + pass "a half-installed hook (failed '${step}') still isolates VPN clients" + else + fail "after failing '${step}' a VPN client is no longer isolated" + fi +done +"${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +# --- a hook that cannot read the uplink's subnet refuses to install ---------- +# Without the uplink subnet only the static ranges cover the exit's own +# network, so an `ip` failure must stop the hook. +real_ip="$(command -v ip)" +cat >"${mock_bin}/ip" <&2 + exit 1 +fi +exec "${real_ip}" "\$@" +MOCK +chmod +x "${mock_bin}/ip" + +if PATH="${mock_bin}:${PATH}" \ + "${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" 2>/dev/null; then + fail "the hook reported success although reading the routes of ${UPLINK} failed" +else + pass "the hook fails when reading the routes of ${UPLINK} fails" +fi +if closed_after_partial_install; then + pass "a hook that could not read the routes of ${UPLINK} still isolates VPN clients" +else + fail "after failing to read the routes of ${UPLINK} a VPN client is no longer isolated" +fi +"${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" +rm -rf "${mock_bin}" + +if [ "${failures}" -ne 0 ]; then + echo "--- final ruleset ---" >&2 + iptables -t filter -S >&2 + iptables -t nat -S >&2 + echo "${failures} check(s) failed" >&2 + exit 1 +fi +echo "all firewall checks passed" diff --git a/test/firewall.sh b/test/firewall.sh new file mode 100755 index 0000000..9baa557 --- /dev/null +++ b/test/firewall.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Checks that a VPN client tunnelled through this exit node can't reach private +# addresses: first that the image runs the hook, then how the rules behave in a +# throwaway privileged container (see test/firewall-check.sh). +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# wg-quick runs PostUp/PreDown from the rendered config, so a template that +# stops calling the hook ships an unfiltered exit node. Without +# --sync-wg-interface gnosis_vpn-server never runs wg-quick at all. +grep -q '^PostUp = /app/wg-hooks.sh up ' gnosisvpn-server/wggvpn.conf.tpl +grep -q '^PreDown = /app/wg-hooks.sh down ' gnosisvpn-server/wggvpn.conf.tpl +grep -q '^COPY .*wg-hooks.sh' gnosisvpn-server/Dockerfile +grep -q -- '--sync-wg-interface' gnosisvpn-server/entrypoint.sh +echo "ok: wggvpn.conf.tpl runs the hook, the image ships it, and the server syncs the interface" +# wg-quick brings wggvpn up before PostUp runs the hook, so FORWARD must be +# DROP before the server starts. +grep -q '^iptables -P FORWARD DROP$' gnosisvpn-server/entrypoint.sh +echo "ok: entrypoint.sh closes FORWARD before the server brings wggvpn up" + +if [ -n "${FIREWALL_TEST_IMAGE:-}" ]; then + # Tests the image's own /app/wg-hooks.sh, bash, iproute2 and iptables, so + # nothing is mounted or installed. --entrypoint skips entrypoint.sh, which + # would start the server. + docker run --rm --privileged --entrypoint bash \ + -v "${PWD}/test/firewall-check.sh:/firewall-check.sh:ro" \ + "${FIREWALL_TEST_IMAGE}" \ + /firewall-check.sh +else + docker run --rm --privileged \ + -v "${PWD}/gnosisvpn-server/wg-hooks.sh:/app/wg-hooks.sh:ro" \ + -v "${PWD}/test/firewall-check.sh:/firewall-check.sh:ro" \ + alpine:3 \ + sh -c 'apk add --no-cache --quiet bash iptables iproute2 ipset >/dev/null && bash /firewall-check.sh' +fi diff --git a/test/traffic-check.sh b/test/traffic-check.sh new file mode 100755 index 0000000..6c1243c --- /dev/null +++ b/test/traffic-check.sh @@ -0,0 +1,359 @@ +#!/bin/bash +# Runs inside a throwaway privileged container (see test/traffic.sh): the +# blocklist and the per-peer shaping the exit node applies to forwarded traffic. +# A veth pair named wggvpn stands in for WireGuard, a second namespace for a VPN +# client, and a third for a public host that answers, so probes are really +# forwarded across wggvpn instead of only checked against a ruleset. +set -euo pipefail + +HOOK=/app/wg-hooks.sh +BLOCKLIST=/app/gvpn-blocklist-update.sh +WG_IF=wggvpn +IFB_IF=ifb-gvpn +BLOCKLIST_SET=gvpn-blocklist +CLIENT_CIDR=10.128.0.0/24 +SERVER_ADDRESS=10.128.0.1 +CLIENT_ADDRESS=10.128.0.2 +# The uplink, and a public host behind it that answers. Both are TEST-NET +# addresses: public as far as the rules are concerned, and never routed. +UPLINK=eth1 +PUBLIC_HOST=203.0.113.9 +PUBLIC_SUBNET=203.0.113.0/24 +# Blocked by the feed below, and not otherwise private, so only the blocklist +# can stop it. +BLOCKED_HOST=203.0.113.77 +CLIENT_NS=vpn-client +CLIENT_IF=veth-client +INTERNET_NS=internet +INTERNET_IF=inet0 +CACHE_DIR=/tmp/blocklist +GOOD_FEED=/tmp/feed-good +SHORT_FEED=/tmp/feed-short +EXTRA_FEED=/tmp/feed-extra + +# What entrypoint.sh derives from WIREGUARD_CIDR and MAX_SLOTS and exports for +# wg-hooks.sh. Rates are the shipped defaults, so tc does not warn about small +# quanta. +export GVPN_PEER_SUBNET_PREFIX=10.128.0 +export GVPN_PEER_MIN_OCTET=2 +export GVPN_PEER_MAX_OCTET=5 +# Deliberately not the shipped defaults, so a rate the setup wizard sets has to +# reach the classes for these checks to pass. +export GVPN_PEER_RATE_KBIT=7000 +export GVPN_PEER_SHAPING_AGGREGATE_KBIT=18000 +export GVPN_BLOCKLIST_CACHE_DIR="${CACHE_DIR}" +export GVPN_BLOCKLIST_MIN_ENTRIES=3 +# Local feeds only: the refresher runs curl, and the sandbox has no route off +# the machine. +export GVPN_BLOCKLIST_FEEDS="file://${GOOD_FEED}" + +failures=0 +pass() { echo "ok: $*"; } +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +setup() { + # A public host behind the uplink that answers pings. + ip netns add "${INTERNET_NS}" + ip link add "${UPLINK}" type veth peer name "${INTERNET_IF}" + ip link set "${INTERNET_IF}" netns "${INTERNET_NS}" + ip addr add 198.51.100.10/24 dev "${UPLINK}" + ip link set "${UPLINK}" up + ip route replace "${PUBLIC_SUBNET}" dev "${UPLINK}" + ip route replace default dev "${UPLINK}" + ip -n "${INTERNET_NS}" addr add 198.51.100.9/24 dev "${INTERNET_IF}" + ip -n "${INTERNET_NS}" addr add "${PUBLIC_HOST}/24" dev "${INTERNET_IF}" + ip -n "${INTERNET_NS}" link set "${INTERNET_IF}" up + ip -n "${INTERNET_NS}" link set lo up + ip -n "${INTERNET_NS}" route add default via 198.51.100.10 + + # The tunnel, and the client behind it. + ip netns add "${CLIENT_NS}" + ip link add "${WG_IF}" type veth peer name "${CLIENT_IF}" + ip link set "${CLIENT_IF}" netns "${CLIENT_NS}" + ip addr add "${SERVER_ADDRESS}/24" dev "${WG_IF}" + ip link set "${WG_IF}" up + ip -n "${CLIENT_NS}" addr add "${CLIENT_ADDRESS}/24" dev "${CLIENT_IF}" + ip -n "${CLIENT_NS}" link set "${CLIENT_IF}" up + ip -n "${CLIENT_NS}" link set lo up + ip -n "${CLIENT_NS}" route add default via "${SERVER_ADDRESS}" + + # Three entries: one that matters (the blocked host) and two that only get + # the feed over the minimum-entry threshold. + cat >"${GOOD_FEED}" <"${SHORT_FEED}" + # A second feed, to check that several feeds are merged. + printf '%s/32\n' 203.0.113.79 >"${EXTRA_FEED}" +} + +# Packet count of the rule in CHAIN whose `iptables -S -v` line contains SPEC, +# with the `-c PACKETS BYTES` counters stripped. Counters are zeroed before +# every probe, so the count belongs to that probe. +counter() { + local chain="$1" spec="$2" + iptables -t filter -S "${chain}" -v | awk -v want="${spec}" -v prefix="-A ${chain} " ' + { + line = $0 + pkts = "missing" + if (match(line, / -c [0-9]+ [0-9]+/)) { + split(substr(line, RSTART + 4, RLENGTH - 4), counts, " ") + pkts = counts[1] + line = substr(line, 1, RSTART - 1) substr(line, RSTART + RLENGTH) + } + # This chain only, so a spec cannot match a rule somewhere else. + if (index(line, prefix) == 1 && index(line, want) > 0) { print pkts; found = 1; exit } + } + END { if (!found) print "missing" } + ' || true +} + +expect_matched() { + local chain="$1" spec="$2" description="$3" pkts + pkts="$(counter "${chain}" "${spec}")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule '${spec}' in ${chain}" + elif [ "${pkts}" -lt 1 ]; then + fail "${description}: rule '${spec}' in ${chain} matched ${pkts} packets" + else + pass "${description}" + fi +} + +expect_unmatched() { + local chain="$1" spec="$2" description="$3" pkts + pkts="$(counter "${chain}" "${spec}")" + if [ "${pkts}" = missing ]; then + fail "${description}: no rule '${spec}' in ${chain}" + elif [ "${pkts}" -ne 0 ]; then + fail "${description}: rule '${spec}' in ${chain} matched ${pkts} packets" + else + pass "${description}" + fi +} + +# probe_from_client DESTINATION: succeeds when the public host answered. +probe_from_client() { + ip netns exec "${CLIENT_NS}" ping -c 1 -W 1 "$1" >/dev/null 2>&1 +} + +zero_counters() { + local chain + for chain in FORWARD INPUT; do + iptables -t filter -Z "${chain}" + done +} + +# `|| true` so a missing set reports as a failed check instead of aborting the +# run under `set -e`. +blocklist_entries() { + ipset list "${BLOCKLIST_SET}" 2>/dev/null | awk '/Number of entries/ { print $4 }' || true +} + +# expect_entries COUNT DESCRIPTION +expect_entries() { + local expected="$1" description="$2" actual + actual="$(blocklist_entries)" + if [ "${actual}" = "${expected}" ]; then + pass "${description}" + else + fail "${description}: set holds '${actual:-nothing}', expected ${expected}" + fi +} + +# Packets sent through class CLASSID of DEV, from `tc -s class show`. +class_packets() { + tc -s class show dev "$1" classid "$2" | awk '/Sent [0-9]+ bytes/ { for (i = 1; i < NF; i++) if ($i == "bytes") { print $(i + 1); exit } }' || true +} + +expect_class_packets() { + local dev="$1" classid="$2" description="$3" pkts + pkts="$(class_packets "${dev}" "${classid}")" + if [ -z "${pkts}" ]; then + fail "${description}: no class ${classid} on ${dev}" + elif [ "${pkts}" -lt 1 ]; then + fail "${description}: class ${classid} on ${dev} saw ${pkts} packets" + else + pass "${description}" + fi +} + +setup + +# --- the hook installs the blocklist drops and the shaping ------------------- +"${HOOK}" up "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +if ipset list "${BLOCKLIST_SET}" >/dev/null 2>&1; then + pass "the hook creates ${BLOCKLIST_SET}, so its DROP rules resolve" +else + fail "the hook did not create ${BLOCKLIST_SET}" +fi + +expect_entries 0 "an empty blocklist blocks nothing" + +# Both directions, and both ahead of the ACCEPT that lets clients out. +# `|| true` throughout: a missing rule must report as a failed check rather +# than abort the run, so a regression shows all of its symptoms at once. +blocked_rule="$(iptables -t filter -S FORWARD | grep -nF -- "-m set --match-set ${BLOCKLIST_SET} dst -j DROP" | cut -d: -f1 || true)" +accepted_rule="$(iptables -t filter -S FORWARD | grep -nF -- "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" | cut -d: -f1 || true)" +if [ -n "${blocked_rule}" ] && [ -n "${accepted_rule}" ] && [ "${blocked_rule}" -lt "${accepted_rule}" ]; then + pass "the client -> blocklisted destination drop precedes the client ACCEPT" +else + fail "the client -> blocklisted destination drop is missing or behind the client ACCEPT" +fi +if iptables -t filter -S FORWARD | grep -qF -- "-o ${WG_IF} -m set --match-set ${BLOCKLIST_SET} src -j DROP"; then + pass "blocklisted sources are dropped on their way back to clients" +else + fail "no FORWARD drop for blocklisted sources returning to clients" +fi + +# --- a refresh replaces the set, atomically, and the cache follows ------------ +if "${BLOCKLIST}" --refresh >/dev/null 2>&1; then + pass "--refresh exits 0" +else + fail "--refresh failed" +fi +expect_entries 3 "a good feed populates the set" +if [ -s "${CACHE_DIR}/blocklist.cidr" ]; then + pass "a good feed is cached for the next boot" +else + fail "no cache file after a successful refresh" +fi + +# --- several feeds are merged, space- or newline-separated ------------------- +GVPN_BLOCKLIST_FEEDS="file://${GOOD_FEED} file://${EXTRA_FEED}" "${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 4 "space-separated feeds are merged" +GVPN_BLOCKLIST_FEEDS="$(printf 'file://%s\nfile://%s' "${GOOD_FEED}" "${EXTRA_FEED}")" \ + "${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 4 "newline-separated feeds are merged" +"${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 3 "the configured feeds replace the merged set" + +zero_counters +if probe_from_client "${BLOCKED_HOST}"; then + fail "the client reached a blocklisted destination" +else + pass "the client cannot reach a blocklisted destination" +fi +expect_matched FORWARD "-m set --match-set ${BLOCKLIST_SET} dst -j DROP" "the blocklist drop matched it" +expect_unmatched FORWARD "-i ${WG_IF} -o ${UPLINK} -j ACCEPT" "the blocklisted packet was not forwarded" + +# --- a feed that is too short is rejected, the previous set stays ------------- +GVPN_BLOCKLIST_FEEDS="file://${SHORT_FEED}" "${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 3 "a feed below the minimum entry count keeps the previous set" + +zero_counters +probe_from_client 203.0.113.78 >/dev/null 2>&1 || true +expect_unmatched FORWARD "-m set --match-set ${BLOCKLIST_SET} dst -j DROP" "the rejected feed's entry is not blocked" + +# --- an unreachable feed falls back to the cache, not to an empty set -------- +GVPN_BLOCKLIST_FEEDS="file:///tmp/does-not-exist" "${BLOCKLIST}" --refresh >/dev/null 2>&1 || true +expect_entries 3 "an unreachable feed falls back to the cached list" + +# --- the cache reloads the set without the network --------------------------- +ipset flush "${BLOCKLIST_SET}" +expect_entries 0 "the set is empty after a flush" +"${BLOCKLIST}" --from-cache >/dev/null 2>&1 || true +expect_entries 3 "--from-cache reloads the cached list" + +# --- short prefixes are accepted for the 224/3 bogon range only --------------- +# ipset normalizes a network, so 224.0.0.0/1 would block 128.0.0.0/1, and a /0 +# fails the whole restore. +parsed="$(printf '%s\n' 224.0.0.0/3 224.0.0.0/1 240.0.0.0/2 240.0.0.0/0 0.0.0.0/1 1.2.3.0/24 \ + | bash -c "source ${BLOCKLIST}; parse_feed" || true)" +if [ "${parsed}" = "$(printf '%s\n' 224.0.0.0/3 1.2.3.0/24)" ]; then + pass "the parser keeps 224.0.0.0/3 and drops every other prefix shorter than /8" +else + fail "the parser kept: ${parsed//$'\n'/ }" +fi + +# --- repeated refreshes in one process leave no temp files -------------------- +# --loop never exits, so its EXIT trap cannot be what cleans up after a refresh. +tmp_root="$(mktemp -d)" +TMPDIR="${tmp_root}" bash -c "source ${BLOCKLIST}; refresh; refresh" >/dev/null 2>&1 || true +leftover="$(find "${tmp_root}" -mindepth 1 | wc -l)" +if [ "${leftover}" -eq 0 ]; then + pass "each refresh removes its temp files" +else + fail "${leftover} temp file(s) left behind by two refreshes" +fi +rm -rf "${tmp_root}" + +# --- a client still reaches public destinations ------------------------------ +zero_counters +if probe_from_client "${PUBLIC_HOST}"; then + pass "the client still reaches a public host" +else + fail "the client could not reach a public host" +fi +expect_unmatched FORWARD "-m set --match-set ${BLOCKLIST_SET} dst -j DROP" "the public host is not blocklisted" + +# --- every peer has a shaped class in both directions ------------------------ +class_count="$(tc class show dev "${WG_IF}" | grep -c 'class htb 1:' || true)" +if [ "${class_count}" -eq 6 ]; then + pass "one class per peer, plus the aggregate and the default class" +else + fail "${class_count} classes on ${WG_IF}, expected 6 (aggregate, default, 4 peers)" +fi +if tc class show dev "${WG_IF}" classid 1:1 | grep -q 'rate 18Mbit'; then + pass "the aggregate class caps the node-wide rate" +else + fail "the aggregate class is not capped at 18Mbit" +fi +if tc class show dev "${WG_IF}" classid 1:2 | grep -q 'ceil 7Mbit'; then + pass "each peer class is capped at the per-peer rate" +else + fail "peer class 1:2 is not capped at 7Mbit" +fi +if [ "$(tc qdisc show dev "${WG_IF}" | grep -c fq_codel)" -eq 5 ]; then + pass "every class has an fq_codel leaf" +else + fail "fq_codel leaves are missing from ${WG_IF}" +fi +if ip link show "${IFB_IF}" >/dev/null 2>&1 && tc class show dev "${IFB_IF}" classid 1:2 | grep -q 'ceil 7Mbit'; then + pass "uploads are shaped on ${IFB_IF}, mirrored from ${WG_IF}" +else + fail "no shaped ${IFB_IF} for upload traffic" +fi + +# The probes above already moved packets through peer .2: the reply left via +# wggvpn (classified by destination) and the request arrived on the mirror +# (classified by source). +expect_class_packets "${WG_IF}" 1:2 "download traffic to peer .2 is shaped" +expect_class_packets "${IFB_IF}" 1:2 "upload traffic from peer .2 is shaped" + +# --- down removes the drops and the shaping, and keeps the set -------------- +"${HOOK}" down "${CLIENT_CIDR}" "${SERVER_ADDRESS}" "${UPLINK}" + +if iptables -t filter -S FORWARD | grep -qF -- "-m set --match-set ${BLOCKLIST_SET}"; then + fail "FORWARD still references ${BLOCKLIST_SET} after down" +else + pass "down removes the blocklist drops" +fi +expect_entries 3 "down keeps the blocklist set, so the refresher keeps working" +if tc qdisc show dev "${WG_IF}" | grep -q 'htb'; then + fail "the shaping tree on ${WG_IF} survived down" +else + pass "down removes the shaping tree" +fi +if ip link show "${IFB_IF}" >/dev/null 2>&1; then + fail "${IFB_IF} survived down" +else + pass "down removes ${IFB_IF}" +fi + +if [ "${failures}" -ne 0 ]; then + echo "--- final state ---" >&2 + iptables -t filter -S >&2 + ipset list "${BLOCKLIST_SET}" >&2 2>/dev/null || true + tc qdisc show dev "${WG_IF}" >&2 + echo "${failures} check(s) failed" >&2 + exit 1 +fi +echo "all traffic checks passed" diff --git a/test/traffic.sh b/test/traffic.sh new file mode 100755 index 0000000..6239bae --- /dev/null +++ b/test/traffic.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Checks the blocklist and the per-peer shaping the exit node applies to +# forwarded traffic: first that the image ships both scripts and wires them up, +# then how they behave in a throwaway privileged container (see +# test/traffic-check.sh). +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# wg-hooks.sh runs the shaping script from PostUp/PreDown, and entrypoint.sh +# loads the cached blocklist before the server starts and refreshes it in the +# background. An image that stopped shipping either would fail here. +grep -q '^COPY .*gvpn-blocklist-update.sh' gnosisvpn-server/Dockerfile +grep -q '^COPY .*gvpn-peer-shaping.sh' gnosisvpn-server/Dockerfile +grep -q 'gvpn-blocklist-update.sh --from-cache' gnosisvpn-server/entrypoint.sh +grep -q 'gvpn-blocklist-update.sh --loop &' gnosisvpn-server/entrypoint.sh +grep -q 'GVPN_SHAPING_BIN:-/app/gvpn-peer-shaping.sh' gnosisvpn-server/wg-hooks.sh + +# The rates and the feeds are set from the DAppNode UI, so their setup-wizard +# fields have to exist. +for field in GVPN_PEER_RATE_KBIT GVPN_PEER_SHAPING_AGGREGATE_KBIT \ + GVPN_BLOCKLIST_FEEDS GVPN_BLOCKLIST_REFRESH_INTERVAL_S; do + if ! grep -qx " name: ${field}" setup-wizard.yml; then + echo "setup-wizard.yml has no ${field} field" >&2 + exit 1 + fi +done +echo "ok: the image ships both scripts, wires them up, and the wizard exposes their settings" + +if [ -n "${TRAFFIC_TEST_IMAGE:-}" ]; then + # Tests the image's own /app, so nothing is mounted or installed. --entrypoint + # skips entrypoint.sh, which would start the server. + docker run --rm --privileged --entrypoint bash \ + -v "${PWD}/test/traffic-check.sh:/traffic-check.sh:ro" \ + "${TRAFFIC_TEST_IMAGE}" \ + /traffic-check.sh +else + docker run --rm --privileged \ + -v "${PWD}/gnosisvpn-server:/app:ro" \ + -v "${PWD}/test/traffic-check.sh:/traffic-check.sh:ro" \ + alpine:3 \ + sh -c 'apk add --no-cache --quiet bash iptables iproute2 ipset curl iputils >/dev/null && bash /traffic-check.sh' +fi