From 0e0355083e32bec6caf1322f99d3177cef25b16f Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Mon, 26 Oct 2026 10:18:33 +0000 Subject: [PATCH 1/5] e2e: Enable more node lifecycle tests on EKS Expand EKS coverage for tests that do not require bink-specific operations: - ControllerMembership: pool membership and daemon placement. - PauseResume: staging while paused and completing after resume. - NonExistingImage: degraded status after a failed image pull. - ControllerRecovery and DaemonRecovery: interrupted rollout recovery. - OperatorRestartResilience: no node disruption after restart and a successful subsequent rollout. Keep SoftReboot bink-only while its EKS failure is investigated, and MidRolloutImageChange bink-only because it is flaky. TagResolution needs local-registry retagging adapted; PullSecretRotation invokes bink; and RebootTimeoutDegraded needs an EKS power-off implementation. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/bootcnode_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 70317d43..902e64ba 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -39,7 +39,7 @@ const ( // BootcNodePool selecting it, and verifies that a BootcNode is created // and the node is labeled bootc.dev/managed. func TestControllerMembership(t *testing.T) { - e2eutil.Providers(t, "bink") + e2eutil.Providers(t, "bink", "eks") t.Parallel() g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) @@ -528,7 +528,7 @@ func getBootCount(t *testing.T, env *e2eutil.Env, ctx context.Context, nodeName // pool paused, verifies the node stages but does not reboot, then resumes // and verifies the update completes. func TestPauseResume(t *testing.T) { - e2eutil.Providers(t, "bink") + e2eutil.Providers(t, "bink", "eks") t.Parallel() g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) @@ -634,7 +634,7 @@ func TestPauseResume(t *testing.T) { // original image, then updates to a non-existing image and verifies the // node enters degraded state and the update does not proceed. func TestNonExistingImage(t *testing.T) { - e2eutil.Providers(t, "bink") + e2eutil.Providers(t, "bink", "eks") t.Parallel() g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) @@ -1010,7 +1010,7 @@ func TestPullSecretRotation(t *testing.T) { // verifies it resumes and completes the interrupted rollout. Covers scenario 5 // of #69 (kill the controller during a roll-out). func TestControllerRecovery(t *testing.T) { - e2eutil.Providers(t, "bink") + e2eutil.Providers(t, "bink", "eks") g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) g.SetDefaultEventuallyPollingInterval(pollInterval) @@ -1211,7 +1211,7 @@ func poolAllUpdated(nodeCount int32, deployedDigest string) types.GomegaMatcher // Covers the daemon half of scenario 5 of #69 (kill the daemon during a // roll-out). func TestDaemonRecovery(t *testing.T) { - e2eutil.Providers(t, "bink") + e2eutil.Providers(t, "bink", "eks") t.Parallel() g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) @@ -1352,7 +1352,7 @@ func TestRebootTimeoutDegraded(t *testing.T) { // TestOperatorUpgrade, which covers an actual version upgrade via a // released manifest rather than a same-version pod restart. func TestOperatorRestartResilience(t *testing.T) { - e2eutil.Providers(t, "bink") + e2eutil.Providers(t, "bink", "eks") g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) g.SetDefaultEventuallyPollingInterval(pollInterval) From 3b50a20f8a026e0cb6f72641aafccd16d5b48468 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Mon, 26 Oct 2026 15:43:26 +0000 Subject: [PATCH 2/5] e2e: Match manifest-list digests in EKS-enabled tests EKS images can report a platform content digest distinct from the manifest-list digest in the configured pullspec. Use the shared digest-aware matcher for recovery, restart, membership, pause/resume and non-existing image assertions, including paused staging checks, so valid images do not time out. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/bootcnode_test.go | 54 +++++++++++++------------------------- test/util/image.go | 19 ++++++++++++++ test/util/image_test.go | 40 ++++++++++++++++++++++++++++ test/util/recovery.go | 10 ++----- 4 files changed, 79 insertions(+), 44 deletions(-) create mode 100644 test/util/image.go create mode 100644 test/util/image_test.go diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 902e64ba..9e0a49c9 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -25,7 +25,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" - "github.com/bootc-dev/bootc-operator/internal/image" "github.com/bootc-dev/bootc-operator/test/e2e/e2eutil" testutil "github.com/bootc-dev/bootc-operator/test/util" ) @@ -93,7 +92,7 @@ func TestControllerMembership(t *testing.T) { testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute, HaveField("Booted", And( HaveField("Image", env.NodeImageDigestedPullSpec()), - HaveField("ImageDigest", env.NodeImageDigest()), + imageMatchesDigest(env.NodeImageDigest()), )), ) @@ -569,19 +568,13 @@ func TestPauseResume(t *testing.T) { err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) return bn.Status, err }).WithTimeout(5 * time.Minute).Should(And( - HaveField("Staged", And( - Not(BeNil()), - HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest())), - )), + HaveField("Staged", imageMatchesDigest(env.NodeImageUpdateDigest())), HaveField("Conditions", ContainElement(And( HaveField("Type", bootcv1alpha1.NodeIdle), HaveField("Status", metav1.ConditionFalse), HaveField("Reason", bootcv1alpha1.NodeReasonStaged), ))), - HaveField("Booted", And( - Not(BeNil()), - HaveField("ImageDigest", Equal(env.NodeImageDigest())), - )), + HaveField("Booted", imageMatchesDigest(env.NodeImageDigest())), )) t.Logf("Node %q staged update but did not reboot (paused)", nodeName) @@ -620,7 +613,7 @@ func TestPauseResume(t *testing.T) { // Phase 5: Wait for node to complete the update — proves the full // update lifecycle completed after resume (reboot, boot into new image). testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, - HaveField("Booted", HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest()))), + HaveField("Booted", imageMatchesDigest(env.NodeImageUpdateDigest())), ) t.Logf("Node %q completed update after resume", nodeName) @@ -677,10 +670,7 @@ func TestNonExistingImage(t *testing.T) { HaveField("Reason", bootcv1alpha1.NodeReasonError), HaveField("Message", ContainSubstring("stage failed")), ))), - HaveField("Booted", And( - Not(BeNil()), - HaveField("ImageDigest", Equal(env.NodeImageDigest())), - )), + HaveField("Booted", imageMatchesDigest(env.NodeImageDigest())), ), "expected node to enter degraded state when pulling non-existing image") t.Logf("Node %q entered degraded state as expected", nodeName) @@ -1059,15 +1049,12 @@ func TestControllerRecovery(t *testing.T) { g.Expect(env.Client.Patch(ctx, unpaused, client.MergeFrom(pool))).To(Succeed()) *pool = *unpaused - g.Consistently(func() (string, error) { + g.Consistently(func() (*bootcv1alpha1.ImageInfo, error) { var bn bootcv1alpha1.BootcNode err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) - if bn.Status.Booted == nil { - return "", err - } - return bn.Status.Booted.ImageDigest, err + return bn.Status.Booted, err }).WithTimeout(20*time.Second).WithPolling(2*time.Second).Should( - Equal(env.NodeImageDigest()), + imageMatchesDigest(env.NodeImageDigest()), "rollout should stall on the original image while the controller is down", ) @@ -1077,7 +1064,7 @@ func TestControllerRecovery(t *testing.T) { t.Logf("Controller restored; waiting for the interrupted rollout to finish") testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 5*time.Minute, - HaveField("Booted", HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest()))), + HaveField("Booted", imageMatchesDigest(env.NodeImageUpdateDigest())), ) t.Logf("Node %q completed the interrupted rollout after controller recovery", nodeName) @@ -1184,9 +1171,7 @@ func fetchEvents( // both the content digest and the manifest-list digest embedded in the // Image pullspec. func imageMatchesDigest(digest string) types.GomegaMatcher { - return WithTransform(func(info *bootcv1alpha1.ImageInfo) bool { - return image.InfoMatchesDigest(info, digest) - }, BeTrue()) + return testutil.ImageMatchesDigest(digest) } func poolAllUpdated(nodeCount int32, deployedDigest string) types.GomegaMatcher { @@ -1226,7 +1211,7 @@ func TestDaemonRecovery(t *testing.T) { pool := env.NewPool("bnp-daemonrec", env.NodeImageDigestedPullSpec()) g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute, - HaveField("Booted", HaveField("ImageDigest", Equal(env.NodeImageDigest()))), + HaveField("Booted", imageMatchesDigest(env.NodeImageDigest())), ) t.Logf("Node %q is Idle with original image", nodeName) @@ -1266,7 +1251,7 @@ func TestDaemonRecovery(t *testing.T) { *pool = *unpaused testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, - HaveField("Booted", HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest()))), + HaveField("Booted", imageMatchesDigest(env.NodeImageUpdateDigest())), ) g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) @@ -1366,7 +1351,7 @@ func TestOperatorRestartResilience(t *testing.T) { pool := env.NewPool("bnp-restartres", env.NodeImageDigestedPullSpec()) g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute, - HaveField("Booted", HaveField("ImageDigest", Equal(env.NodeImageDigest()))), + HaveField("Booted", imageMatchesDigest(env.NodeImageDigest())), ) g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). Should(poolAllUpdated(1, env.NodeImageDigest())) @@ -1380,15 +1365,12 @@ func TestOperatorRestartResilience(t *testing.T) { // Phase 3: The managed node must not be disrupted by the operator restart: // it must stay on its current image without rebooting. - g.Consistently(func() (string, error) { + g.Consistently(func() (*bootcv1alpha1.ImageInfo, error) { var bn bootcv1alpha1.BootcNode err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) - if bn.Status.Booted == nil { - return "", err - } - return bn.Status.Booted.ImageDigest, err + return bn.Status.Booted, err }).WithTimeout(30*time.Second).WithPolling(3*time.Second).Should( - Equal(env.NodeImageDigest()), + imageMatchesDigest(env.NodeImageDigest()), "node should stay on its image across an operator restart", ) @@ -1397,7 +1379,7 @@ func TestOperatorRestartResilience(t *testing.T) { "operator restart must not reboot the node") testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 2*time.Minute, - HaveField("Booted", HaveField("ImageDigest", Equal(env.NodeImageDigest()))), + HaveField("Booted", imageMatchesDigest(env.NodeImageDigest())), ) g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). Should(poolAllUpdated(1, env.NodeImageDigest())) @@ -1414,7 +1396,7 @@ func TestOperatorRestartResilience(t *testing.T) { t.Logf("Patched pool to update image %s after operator restart", updateRef) testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 5*time.Minute, - HaveField("Booted", HaveField("ImageDigest", Equal(env.NodeImageUpdateDigest()))), + HaveField("Booted", imageMatchesDigest(env.NodeImageUpdateDigest())), ) g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) diff --git a/test/util/image.go b/test/util/image.go new file mode 100644 index 00000000..bcdc2fe9 --- /dev/null +++ b/test/util/image.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "github.com/onsi/gomega" + "github.com/onsi/gomega/types" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/internal/image" +) + +// ImageMatchesDigest matches either the content digest or the manifest-list +// digest embedded in an ImageInfo's pullspec. +func ImageMatchesDigest(digest string) types.GomegaMatcher { + return gomega.WithTransform(func(info *bootcv1alpha1.ImageInfo) bool { + return image.InfoMatchesDigest(info, digest) + }, gomega.BeTrue()) +} diff --git a/test/util/image_test.go b/test/util/image_test.go new file mode 100644 index 00000000..db1b84ef --- /dev/null +++ b/test/util/image_test.go @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "strings" + "testing" + + . "github.com/onsi/gomega" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" +) + +func TestImageMatchesDigest(t *testing.T) { + contentDigest := "sha256:" + strings.Repeat("a", 64) + listDigest := "sha256:" + strings.Repeat("b", 64) + info := &bootcv1alpha1.ImageInfo{ + Image: "example.com/node@" + listDigest, + ImageDigest: contentDigest, + } + + for _, tt := range []struct { + name string + info *bootcv1alpha1.ImageInfo + digest string + want bool + }{ + {"content digest", info, contentDigest, true}, + {"manifest-list digest", info, listDigest, true}, + {"different digest", info, "sha256:" + strings.Repeat("c", 64), false}, + {"missing image", nil, listDigest, false}, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + matched, err := ImageMatchesDigest(tt.digest).Match(tt.info) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(matched).To(Equal(tt.want)) + }) + } +} diff --git a/test/util/recovery.go b/test/util/recovery.go index 8c5879be..16fd5939 100644 --- a/test/util/recovery.go +++ b/test/util/recovery.go @@ -47,14 +47,8 @@ func StagePausedUpdate( err := c.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) return bn.Status, err }).WithTimeout(5*time.Minute).Should(gomega.And( - gomega.HaveField("Staged", gomega.And( - gomega.Not(gomega.BeNil()), - gomega.HaveField("ImageDigest", gomega.Equal(targetDigest)), - )), - gomega.HaveField("Booted", gomega.And( - gomega.Not(gomega.BeNil()), - gomega.HaveField("ImageDigest", gomega.Equal(originalDigest)), - )), + gomega.HaveField("Staged", ImageMatchesDigest(targetDigest)), + gomega.HaveField("Booted", ImageMatchesDigest(originalDigest)), gomega.HaveField("Conditions", gomega.ContainElement(gomega.And( gomega.HaveField("Type", bootcv1alpha1.NodeIdle), gomega.HaveField("Status", metav1.ConditionFalse), From 0d4de36cbac915f5e61e084c862f084017a7d050 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Tue, 27 Oct 2026 09:01:51 +0100 Subject: [PATCH 3/5] make: add dependency between eks test and helm The helm chart were missing before the eks test running. Signed-off-by: Alice Frosi --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 88fe837a..b3e3d949 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,7 @@ EKS_IMAGE_PULL_SECRET ?= EKS_NODE_PULL_SECRET ?= .PHONY: e2e-eks -e2e-eks: ## Run e2e tests against EKS. V=1 for verbose. RUN= to filter. +e2e-eks: helm ## Run e2e tests against EKS. V=1 for verbose. RUN= to filter. rm -rf $(ARTIFACTS) cd test/e2e && KUBECONFIG="$(KUBECONFIG)" \ E2E_PROVIDER=eks \ From 20607d74573d9220ddb264be61e7a85106c430c1 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Wed, 28 Oct 2026 09:57:41 +0000 Subject: [PATCH 4/5] e2e: Match membership node images by digest EKS nodes can report a tag-based booted image reference even when the pool uses a digest-qualified reference. Check image identity by digest rather than requiring identical pullspec spelling. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/bootcnode_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 9e0a49c9..4fdb712a 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -90,10 +90,7 @@ func TestControllerMembership(t *testing.T) { )), "expected exactly one running daemon pod on %s", nodeName) testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute, - HaveField("Booted", And( - HaveField("Image", env.NodeImageDigestedPullSpec()), - imageMatchesDigest(env.NodeImageDigest()), - )), + HaveField("Booted", imageMatchesDigest(env.NodeImageDigest())), ) // Verify pool status reflects steady state. From 38c527affd40596a64a1116ee5a7bc315c9af19e Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Wed, 28 Oct 2026 11:56:43 +0000 Subject: [PATCH 5/5] e2e: Increase restart resilience rollout timeout to eight minutes Allow more time for post-reboot networking and daemon startup on EKS. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/bootcnode_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 4fdb712a..2f5cdbbf 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -1392,7 +1392,7 @@ func TestOperatorRestartResilience(t *testing.T) { t.Logf("Patched pool to update image %s after operator restart", updateRef) - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 5*time.Minute, + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, HaveField("Booted", imageMatchesDigest(env.NodeImageUpdateDigest())), ) g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).