From 611b3184f42244d67dab9c2b5996d6c2f87980a1 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 22:01:02 +0100 Subject: [PATCH 1/2] fix(cluster): run the join wait from the image's filesystem Docker mounts a service's tmpfs noexec, and k3s finds tailscale with Go's exec.LookPath, which passes over a file it may not execute, so the wrapper under /run was never run and a joining node on Tailscale's control plane still never joined. The wrapper now lives under /usr/local/lib/github-runner-node/bin, is removed again when the node joins nothing, and logs a line to the container's log after its wait, which the mesh integration test now requires of every joining server. Fixes #116 --- .../files/k3s/node-entrypoint.sh | 9 +++++-- tests/mesh/run.sh | 7 ++++++ tests/unit/test_cluster_join_wait.py | 24 +++++++++++++++---- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index 6d02a1c0..07737786 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -100,10 +100,13 @@ wait_for_mesh_host() { fi } -# Writes a `tailscale` into $2, which k3s is started with first on its PATH (tailscale_up_wait_dir below), that runs the real one ($3) for every command unchanged and, after `tailscale up`, waits as wait_for_mesh_host does for $1, the host this node joins. k3s's own --vpn-auth start (StartVPN in k3s's pkg/vpn/vpn.go) runs `tailscale up --authkey ... --timeout=30s --reset` on every start, which restarts the Tailscale backend from its cached network map, and k3s dials the join host as soon as that command returns. Confirmed live on Tailscale's own control plane with Tailscale v1.102.4: the backend reloaded a cached map with no peers ("Start: loaded netmap from disk cache; 0 peers"), so the join host was neither a peer nor a MagicDNS name for about a second, k3s exited with "no such host", and every restart of the container repeated it, so the node never joined. The wait above, before k3s starts, cannot cover it, since the reset comes after it. $2 is under /run, a tmpfs in docker-compose.yml, so the wrapper is written afresh at every start of the container and never stands in front of the entrypoint's own tailscale commands; it follows the entrypoint, so it needs no image rebuild, and a node that joins nothing gets none. -tailscale_up_wait_dir=/run/github-runner-node/bin +# Writes a `tailscale` into $2, which k3s is started with first on its PATH (tailscale_up_wait_dir below), that runs the real one ($3) for every command unchanged and, after `tailscale up`, waits as wait_for_mesh_host does for $1, the host this node joins, then says so in the container's log. k3s's own --vpn-auth start (StartVPN in k3s's pkg/vpn/vpn.go) runs `tailscale up --authkey ... --timeout=30s --reset` on every start, which restarts the Tailscale backend from its cached network map, and k3s dials the join host as soon as that command returns. Confirmed live on Tailscale's own control plane with Tailscale v1.102.4: the backend reloaded a cached map with no peers ("Start: loaded netmap from disk cache; 0 peers"), so the join host was neither a peer nor a MagicDNS name for about a second, k3s exited with "no such host", and every restart of the container repeated it, so the node never joined. The wait above, before k3s starts, cannot cover it, since the reset comes after it. +# +# $2 is on the image's own filesystem, never under the service's tmpfs mounts: Docker mounts a tmpfs noexec unless told otherwise, and k3s finds the command with Go's exec.LookPath, which passes over a file it may not execute, so a wrapper under /run was never run (confirmed live: the node kept failing as before, with the wrapper in place). The container's filesystem outlives a restart, so the wrapper is written afresh at every start, and removed when the node joins nothing, so a wrapper from an earlier configuration never waits for a host this node no longer joins. Only k3s has $2 on its PATH, so the entrypoint's own tailscale commands run the real one. +tailscale_up_wait_dir=/usr/local/lib/github-runner-node/bin install_tailscale_up_wait() { if [ -z "$1" ]; then + rm -f "$2/tailscale" return 0 fi mkdir -p "$2" @@ -114,6 +117,8 @@ install_tailscale_up_wait() { # shellcheck disable=SC2016 # Written into the wrapper unexpanded, for the wrapper to expand. echo 'if [ "${1:-}" = up ]; then' printf " until '%s' ip -4 '%s' >/dev/null 2>&1 && nslookup '%s' 100.100.100.100 >/dev/null 2>&1; do sleep 0.5; done\n" "$3" "$1" "$1" + # The container's log is PID 1's standard error, k3s's own once the entrypoint has exec'd it; k3s keeps this command's output to itself. Best effort: the line is for whoever reads the log (and the mesh integration test), and failing to write it must not fail k3s's tailscale up. + printf " echo 'github-runner-node: %s is a peer and resolves after tailscale up' 2>/dev/null > /proc/1/fd/2 || true\n" "$1" echo 'fi' } > "$2/tailscale" chmod 0755 "$2/tailscale" diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 0f8f88f5..f5ce887c 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -201,6 +201,13 @@ assert_cluster() { done kubectl_on 1 get nodes -o wide + # The entrypoint's wrapper around k3s's own `tailscale up` logs this line after it has waited for the join host; its absence means k3s never ran the wrapper (as when it sat on a noexec tmpfs, where exec.LookPath passes over it), whether or not this control server needed the wait. + log "Checking each joining server's k3s ran the wait after its own tailscale up" + for index in $(seq 2 "$node_count"); do + docker logs "$(container "$index")" 2>&1 | grep -qF "github-runner-node: $(node_name 1).mesh.grtest.internal is a peer and resolves after tailscale up" \ + || fail "$(container "$index") never logged the wait after k3s's tailscale up" + done + log "Checking the servers were chosen automatically (all three) and etcd has three members" nodes=$(kubectl_on 1 get nodes -l node-role.kubernetes.io/etcd=true --no-headers | wc -l | tr -d ' ') [ "$nodes" = "$node_count" ] || fail "expected ${node_count} etcd nodes, found ${nodes}" diff --git a/tests/unit/test_cluster_join_wait.py b/tests/unit/test_cluster_join_wait.py index 203be653..f4bf7b64 100644 --- a/tests/unit/test_cluster_join_wait.py +++ b/tests/unit/test_cluster_join_wait.py @@ -58,7 +58,7 @@ def setUp(self) -> None: nslookup.write_text(NSLOOKUP_STUB) nslookup.chmod(0o755) (self.stub / "peer-after").write_text("3") - self.wrapper_dir = self.work / "run" / "github-runner-node" / "bin" + self.wrapper_dir = self.work / "usr" / "local" / "lib" / "github-runner-node" / "bin" def install(self, host: str = HOST) -> subprocess.CompletedProcess[str]: script = "\n".join(["set -u", function_source("install_tailscale_up_wait"), f'install_tailscale_up_wait "{host}" "{self.wrapper_dir}" "{self.real}"']) @@ -80,6 +80,13 @@ def test_up_runs_unchanged_then_waits_until_the_join_host_is_a_peer_and_resolves ["up --authkey example --timeout=30s --reset", f"ip -4 {HOST}", f"ip -4 {HOST}", f"ip -4 {HOST}", f"nslookup {HOST} 100.100.100.100"], ) + def test_the_wait_is_written_to_the_container_s_log_without_failing_when_it_cannot_be(self) -> None: + # Outside a container /proc/1/fd/2 is not this test's to write, which is the case the wrapper must survive; inside one it is the container's log, where the mesh integration test finds the line. + self.install() + self.assertIn(f"echo 'github-runner-node: {HOST} is a peer and resolves after tailscale up' 2>/dev/null > /proc/1/fd/2 || true", (self.wrapper_dir / "tailscale").read_text()) + result = self.run_wrapper("up") + self.assertEqual(result.returncode, 0, result.stderr) + def test_any_other_command_runs_unchanged_and_waits_for_nothing(self) -> None: self.install() result = self.run_wrapper("status", "--json") @@ -97,6 +104,12 @@ def test_a_node_that_joins_nothing_gets_no_wrapper(self) -> None: self.assertEqual(self.install(host="").returncode, 0) self.assertFalse(self.wrapper_dir.exists()) + def test_a_wrapper_left_by_an_earlier_start_is_removed_when_the_node_joins_nothing_now(self) -> None: + # The container's filesystem outlives a restart, so without this a node whose configuration no longer joins anything would wait for its old join host. + self.install() + self.assertEqual(self.install(host="").returncode, 0) + self.assertFalse((self.wrapper_dir / "tailscale").exists()) + class JoinHostTest(unittest.TestCase): def join_host(self, url: str) -> str: @@ -127,13 +140,16 @@ def test_an_agent_waits_for_the_server_it_dials(self) -> None: self.assertIn('install_tailscale_up_wait "$agent_join_host" "$tailscale_up_wait_dir" /usr/local/bin/tailscale', body) self.assertIn('PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s agent', body) - def test_the_wrapper_lives_on_a_tmpfs_of_both_services_so_every_start_writes_it_afresh(self) -> None: + def test_the_wrapper_is_on_the_image_s_filesystem_where_it_may_be_executed(self) -> None: + # Docker mounts a tmpfs noexec, and Go's exec.LookPath passes over a file it may not execute, so a wrapper under a tmpfs or a mount is never run. A mount's options are not visible here, so the directory is kept clear of every mount either service has. directory = re.search(r"^tailscale_up_wait_dir=(\S+)$", ENTRYPOINT.read_text(), re.MULTILINE) assert directory is not None services = yaml.safe_load(COMPOSE.read_text())["services"] for name in ("k3s", "k3s-agent"): - with self.subTest(service=name): - self.assertTrue(any(directory.group(1).startswith(mount.rstrip("/") + "/") for mount in services[name]["tmpfs"])) + mounts = [*services[name]["tmpfs"], *(volume.split(":")[1] for volume in services[name]["volumes"])] + for mount in mounts: + with self.subTest(service=name, mount=mount): + self.assertFalse((directory.group(1) + "/").startswith(mount.rstrip("/") + "/")) if __name__ == "__main__": From fa6374dcc465f27d2cd97f45ea9641c7cfda59c5 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 22:18:11 +0100 Subject: [PATCH 2/2] test(mesh): read a joining server's whole log for the wait's line grep -q stops at the first match, and under pipefail the docker logs it cut off with a broken pipe failed the check even when the line was there. --- tests/mesh/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index f5ce887c..ea89b248 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -204,7 +204,8 @@ assert_cluster() { # The entrypoint's wrapper around k3s's own `tailscale up` logs this line after it has waited for the join host; its absence means k3s never ran the wrapper (as when it sat on a noexec tmpfs, where exec.LookPath passes over it), whether or not this control server needed the wait. log "Checking each joining server's k3s ran the wait after its own tailscale up" for index in $(seq 2 "$node_count"); do - docker logs "$(container "$index")" 2>&1 | grep -qF "github-runner-node: $(node_name 1).mesh.grtest.internal is a peer and resolves after tailscale up" \ + # grep without -q reads the whole log: with -q it would stop at the first match and, under pipefail, the docker logs it cut off would fail the check. + docker logs "$(container "$index")" 2>&1 | grep -F "github-runner-node: $(node_name 1).mesh.grtest.internal is a peer and resolves after tailscale up" >/dev/null \ || fail "$(container "$index") never logged the wait after k3s's tailscale up" done