From 4fb04e638d3424cb33af3e79694defcaee31d9a8 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Mon, 22 Jun 2026 16:17:46 -0700 Subject: [PATCH] test: add Eventually polling helper for test setup waits The keystone auth suite installs cert-manager and trust-manager with helm --wait and then immediately applies Issuer, Certificate, and Bundle resources. helm --wait returns when the webhook deployments report ready, but the webhook service endpoints may not be programmed on the apiserver's node yet, so the apply is rejected with 'failed calling webhook ... connect: connection refused' and the suite fails before any test runs. This is currently the dominant failure mode of the keystone suite, seen on master and on PRs that cannot have caused it (dependabot github-actions bumps, mergify backports). Add utils.Eventually, a timeout-based polling primitive intended as the shared replacement for hand-rolled retry loops in test code: - cond is func(ctx) error rather than func() bool, so each failed attempt logs its reason via t.Logf and the final timeout error wraps the last attempt's error. - cond runs on the calling goroutine and receives a context carrying the overall deadline, so cooperative operations stop at the deadline instead of overrunning it. It is never run on another goroutine: a hung attempt would keep executing concurrently with its retry, and require's FailNow is unsafe off the test goroutine. - utils.AttemptTimeout decorates a cond with a per-attempt deadline for operations that can hang but would succeed if canceled and retried. The bound is cooperative; conds that cannot honor a context must be bounded at the operation level instead. Expose it as a K8sHelper.ApplyWithRetry method that retries kubectl apply until the webhooks accept connections, and use it for the keystone setup applies instead of failing the suite on the first attempt. Signed-off-by: Joshua Hoblitt --- tests/framework/utils/eventually.go | 93 +++++++++++++++++++ tests/framework/utils/eventually_test.go | 96 ++++++++++++++++++++ tests/framework/utils/k8s_helper.go | 20 ++++ tests/integration/ceph_base_keystone_test.go | 10 +- 4 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 tests/framework/utils/eventually.go create mode 100644 tests/framework/utils/eventually_test.go diff --git a/tests/framework/utils/eventually.go b/tests/framework/utils/eventually.go new file mode 100644 index 000000000..0268b75d3 --- /dev/null +++ b/tests/framework/utils/eventually.go @@ -0,0 +1,93 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "testing" + "time" +) + +// eventuallyPollInterval is the cadence of Eventually. It is deliberately not +// a parameter: consumers poll cheap idempotent operations (kubectl applies, +// rgw admin / S3 reads), and uniform call sites beat another knob. Add a +// variant if a real need for a different cadence ever appears. +const eventuallyPollInterval = time.Second + +// Eventually polls cond on the calling goroutine every second until it +// returns nil, the timeout elapses, or ctx is canceled. Each failed attempt +// is logged via t.Logf so CI output shows liveness attributed to the right +// (sub)test, and the last attempt's error is wrapped into the returned error. +// +// The context passed to cond is derived from ctx with the overall timeout +// applied, so a cond that honors it stops at the deadline instead of +// overrunning it. To additionally bound each attempt, wrap cond with +// AttemptTimeout. +// +// cond must not call assert or require: report transient failures by +// returning an error, and make assertions on captured state after the wait. +// Eventually never runs cond on another goroutine — a hung attempt would keep +// executing concurrently with its retry, and require's FailNow is unsafe off +// the test goroutine (which is also why testify's assert.Eventually is not +// used here). +func Eventually(ctx context.Context, t *testing.T, timeout time.Duration, desc string, cond func(context.Context) error) error { + t.Helper() + + loopCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + for attempt := 1; ; attempt++ { + err := cond(loopCtx) + if err == nil { + return nil + } + t.Logf("waiting for %s (attempt %d): %v", desc, attempt, err) + + select { + case <-loopCtx.Done(): + if cerr := ctx.Err(); cerr != nil { + return fmt.Errorf("canceled while waiting for %s: %w (last error: %v)", desc, cerr, err) + } + return fmt.Errorf("condition not met within %s: %s: %w", timeout, desc, err) + case <-time.After(eventuallyPollInterval): + } + } +} + +// AttemptTimeout bounds each attempt of cond with its own deadline, for +// operations that can hang but would succeed if canceled and retried. The +// bound is cooperative: cond only stops early if it honors the context it is +// given (requests built with http.NewRequestWithContext, exec.CommandContext, +// SDK *WithContext calls, ...). A cond that ignores its context is unaffected +// by AttemptTimeout: +// +// // WRONG: http.Get ignores ctx, so nothing cancels a hung fetch +// utils.AttemptTimeout(10*time.Second, func(ctx context.Context) error { +// _, err := http.Get(url) +// return err +// }) +// +// Operations that cannot honor a context must be bounded at the operation +// level instead (e.g. http.Client{Timeout: ...}). +func AttemptTimeout(d time.Duration, cond func(context.Context) error) func(context.Context) error { + return func(ctx context.Context) error { + attemptCtx, cancel := context.WithTimeout(ctx, d) + defer cancel() + return cond(attemptCtx) + } +} diff --git a/tests/framework/utils/eventually_test.go b/tests/framework/utils/eventually_test.go new file mode 100644 index 000000000..ffaf7d8e6 --- /dev/null +++ b/tests/framework/utils/eventually_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEventuallySucceedsImmediately(t *testing.T) { + calls := 0 + err := Eventually(context.Background(), t, time.Minute, "immediate success", func(context.Context) error { + calls++ + return nil + }) + require.NoError(t, err) + assert.Equal(t, 1, calls) +} + +func TestEventuallyRetriesUntilSuccess(t *testing.T) { + calls := 0 + err := Eventually(context.Background(), t, time.Minute, "second attempt succeeds", func(context.Context) error { + calls++ + if calls < 2 { + return errors.New("not yet") + } + return nil + }) + require.NoError(t, err) + assert.Equal(t, 2, calls) +} + +func TestEventuallyTimeoutWrapsLastError(t *testing.T) { + cause := errors.New("still broken") + err := Eventually(context.Background(), t, 10*time.Millisecond, "never succeeds", func(context.Context) error { + return cause + }) + require.Error(t, err) + assert.ErrorIs(t, err, cause) + assert.Contains(t, err.Error(), "never succeeds") +} + +func TestEventuallyParentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := Eventually(ctx, t, time.Minute, "canceled parent", func(context.Context) error { + return errors.New("not yet") + }) + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestEventuallyCondSeesOverallDeadline(t *testing.T) { + err := Eventually(context.Background(), t, 10*time.Millisecond, "cond honors ctx", func(ctx context.Context) error { + // a cooperative operation blocks only until the loop deadline + <-ctx.Done() + return ctx.Err() + }) + require.Error(t, err) + assert.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestAttemptTimeoutCancelsHungAttempt(t *testing.T) { + calls := 0 + err := Eventually(context.Background(), t, time.Minute, "hung attempt is canceled and retried", + AttemptTimeout(10*time.Millisecond, func(ctx context.Context) error { + calls++ + if calls < 2 { + // simulate an operation that hangs until canceled + <-ctx.Done() + return ctx.Err() + } + return nil + })) + require.NoError(t, err) + assert.Equal(t, 2, calls) +} diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 06fa218ca..ffdb9f5c9 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -237,6 +237,26 @@ func (k8sh *K8sHelper) ResourceOperation(action string, manifest string) error { return fmt.Errorf("could Not create resource in args : %v -- %v", args, err) } +// applyRetryTimeout bounds ApplyWithRetry. A webhook backend (cert-manager, +// trust-manager, ...) is typically reachable within seconds of its deployment +// reporting ready; two minutes sits comfortably above that without hanging a +// genuinely broken setup for long. +const applyRetryTimeout = 2 * time.Minute + +// ApplyWithRetry retries "kubectl apply" of manifest until it succeeds or +// applyRetryTimeout elapses. It exists for resources guarded by an admission +// webhook whose backing service may not be reachable the instant its +// deployment reports ready (e.g. cert-manager / trust-manager): "helm --wait" +// returns once the webhook deployment is ready, but the first apply can still +// fail with "failed calling webhook ... connection refused". description names +// the resource in the retry log lines. See Eventually for polling semantics. +func (k8sh *K8sHelper) ApplyWithRetry(ctx context.Context, manifest, description string) error { + return Eventually(ctx, k8sh.T(), applyRetryTimeout, "apply "+description, + func(context.Context) error { + return k8sh.ResourceOperation("apply", manifest) + }) +} + // DeletePod performs a kubectl delete pod on the given pod func (k8sh *K8sHelper) DeletePod(namespace, name string) error { args := append([]string{"--grace-period=0", "pod"}, name) diff --git a/tests/integration/ceph_base_keystone_test.go b/tests/integration/ceph_base_keystone_test.go index b374f7f5d..bcb7f5dff 100644 --- a/tests/integration/ceph_base_keystone_test.go +++ b/tests/integration/ceph_base_keystone_test.go @@ -114,27 +114,27 @@ func InstallKeystoneInTestCluster(shelper *utils.K8sHelper, namespace string) er return err } - if err := shelper.ResourceOperation("apply", keystoneApiClusterIssuer(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, keystoneApiClusterIssuer(namespace), "ClusterIssuer"); err != nil { logger.Errorf("Could not apply ClusterIssuer in namespace %s: %s", namespace, err) return err } - if err := shelper.ResourceOperation("apply", keystoneApiCaCertificate(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, keystoneApiCaCertificate(namespace), "CA Certificate"); err != nil { logger.Errorf("Could not apply ClusterIssuer CA Certificate in namespace %s: %s", namespace, err) return err } - if err := shelper.ResourceOperation("apply", keystoneApiCaIssuer(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, keystoneApiCaIssuer(namespace), "CA Issuer"); err != nil { logger.Errorf("Could not install CA Issuer in namespace %s: %s", namespace, err) return err } - if err := shelper.ResourceOperation("apply", keystoneApiCertificate(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, keystoneApiCertificate(namespace), "Certificate"); err != nil { logger.Errorf("Could not create Certificate (request) in namespace %s", namespace) return err } - if err := shelper.ResourceOperation("apply", trustManagerBundle(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, trustManagerBundle(namespace), "trust-manager Bundle"); err != nil { logger.Errorf("Could not create CA Certificate Bundle in namespace %s", namespace) return err }