From 70772e350af2c0dd1b65bbcc0d10f22944c28b47 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 20 Oct 2026 08:30:00 +0000 Subject: [PATCH 1/2] fix(ci): keep the GPU reclaim decoys alive on multicall coreutils ubuntu-latest now resolves to Ubuntu 26.04, which ships uutils coreutils. Its /bin/sleep is one multicall binary that picks the tool to run from argv[0], so the decoys made by copying it to .../llama-server print "coreutils: unknown program 'llama-server'" and exit at once. The reclaim self-test then fails three checks and the xtask contract test that runs the real --assert-clear passes a scan it should fail, which turns build-and-test, Coverage and Test (affected crates) red for every run on the new image. Copy bash instead: it ignores the name it runs under, so argv[0] stays the real-shaped path the rule is meant to match. The body is a loop because bash execs a lone `sleep 300` in place, which would put `sleep` back in argv[0]. Its `sleep 1` children name no E2E root, so they are never selected, and SIGTERM still ends the decoy. Verified with the self-test (plain and with TMPDIR redirected) and the xtask test in ubuntu:26.04 and ubuntu:24.04 containers; both failed on 26.04 before this change. Signed-off-by: Roman Inflianskas --- scripts/reclaim-gpu.sh | 21 +++++++++++++++------ xtask/src/workflow_contract.rs | 12 ++++++++---- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 87db7a7b4..37406c3b2 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -452,13 +452,21 @@ assert_clear() { # Spawn a decoy whose argv[0] is a real-shaped path, so the match is tested # against the same string a live engine process would present. +# +# The executable is a copy of bash, not of sleep: where coreutils is a multicall +# binary (uutils, the default since Ubuntu 25.10), /bin/sleep picks the tool to +# run from argv[0], so a copy named llama-server prints "unknown program" and +# exits before the self-test can see it. bash ignores the name it runs under. +# The body is a loop, not `sleep 300`, because bash execs a lone simple command +# in place and argv[0] would become `sleep`. Its `sleep 1` children name no E2E +# root, so they are never selected, and a TERM still ends the decoy. spawn_decoy() { local path="$1" mkdir -p "$(dirname "${path}")" - cp /bin/sleep "${path}" + cp "$(command -v bash)" "${path}" # Detach stdio: this runs inside a command substitution, and a background # child holding the capture pipe open would block the caller until it exits. - "${path}" 300 >/dev/null 2>&1 & + "${path}" -c 'for _ in {1..300}; do sleep 1; done' >/dev/null 2>&1 & echo $! } @@ -504,7 +512,7 @@ spawn_zombie() { } # Spawn a decoy that IGNORES SIGTERM, so the TERM -> grace -> KILL escalation is -# exercised. A plain `cp /bin/sleep` decoy dies on the first TERM and leaves the +# exercised. A plain spawn_decoy decoy dies on the first TERM and leaves the # escalation branch unreached, which is how it went untested. spawn_stubborn_decoy() { local path="$1" @@ -524,9 +532,10 @@ DECOY # Spawn a decoy carrying the record delimiters INSIDE one argument — a newline # and a tab, as a `--chat-template` or `--prompt` value does. # -# Not spawn_decoy with an extra argument: that decoy is `cp /bin/sleep`, and -# sleep rejects a non-numeric argument and exits before it can be observed. This -# needs something that ignores what it is handed and stays alive. +# Not spawn_decoy with an extra argument: that decoy is `bash -c `, and +# bash reads the argument after the loop as $0, which is not what lands in the +# record under test. This needs something that ignores what it is handed and +# stays alive. # # Ordinary in every other respect — it must die on SIGTERM, so it is the stubborn # decoy's body without the trap. Its `sleep 1` children name no E2E root, so they diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index c6389d372..ade59c2dd 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2897,12 +2897,16 @@ fi // argv[0] names an E2E root and an engine marker, which is what the // rule matches, and sits under the scope so nothing else can. + // + // A copy of bash running a loop, for the reasons `spawn_decoy` in + // reclaim-gpu.sh gives: a copied multicall `sleep` exits at once as an + // unknown program, and bash would exec a lone `sleep 300` in place. let decoy_dir = scope.path().join("e2e-prewarm/bin"); std::fs::create_dir_all(&decoy_dir).expect("decoy dir"); let decoy = decoy_dir.join("llama-server"); - std::fs::copy("/bin/sleep", &decoy).expect("copy /bin/sleep as the decoy"); + std::fs::copy("/bin/bash", &decoy).expect("copy /bin/bash as the decoy"); let mut child = std::process::Command::new(&decoy) - .arg("300") + .args(["-c", "for _ in {1..300}; do sleep 1; done"]) .spawn() .expect("spawn the decoy engine process"); let decoy_pid = child.id(); @@ -2913,8 +2917,8 @@ fi &env, &reclaim, ); - // Reaped before any assertion: a panic here must not leave a `sleep - // 300` behind on a self-hosted runner, of all things. + // Reaped before any assertion: a panic here must not leave a 300s + // decoy behind on a self-hosted runner, of all things. let _ = child.kill(); let _ = child.wait(); let (code, log) = got; From 32cf46e2e09217ebaa6a0a6abd6218a706b665fe Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 20 Oct 2026 08:55:25 +0000 Subject: [PATCH 2/2] fix(ci): correct the delimiter decoy's reason and detach the xtask decoy The delimiter decoy's comment said bash would swallow an extra argument as $0; it does not, and the argument reaches /proc//cmdline. It is a separate helper only because spawn_decoy passes no arguments through. Detach the xtask test's decoy from the test's stdio, as the script's decoy already is: its sleep 1 child outlives the decoy's SIGKILL by up to a second and would hold the inherited output open meanwhile. Signed-off-by: Roman Inflianskas --- scripts/reclaim-gpu.sh | 6 ++---- xtask/src/workflow_contract.rs | 6 ++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 37406c3b2..09a89d2e2 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -532,10 +532,8 @@ DECOY # Spawn a decoy carrying the record delimiters INSIDE one argument — a newline # and a tab, as a `--chat-template` or `--prompt` value does. # -# Not spawn_decoy with an extra argument: that decoy is `bash -c `, and -# bash reads the argument after the loop as $0, which is not what lands in the -# record under test. This needs something that ignores what it is handed and -# stays alive. +# A helper of its own because spawn_decoy passes no arguments through. Like that +# decoy, it ignores what it is handed and stays alive. # # Ordinary in every other respect — it must die on SIGTERM, so it is the stubborn # decoy's body without the trap. Its `sleep 1` children name no E2E root, so they diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index ade59c2dd..d0810deaa 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2907,6 +2907,12 @@ fi std::fs::copy("/bin/bash", &decoy).expect("copy /bin/bash as the decoy"); let mut child = std::process::Command::new(&decoy) .args(["-c", "for _ in {1..300}; do sleep 1; done"]) + // Detached, as in the script: a `sleep 1` child outlives the + // SIGKILL below by up to a second and would hold the test's + // inherited stdio open meanwhile. + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) .spawn() .expect("spawn the decoy engine process"); let decoy_pid = child.id();