diff --git a/tests/framework/utils/eventually.go b/tests/framework/utils/eventually.go new file mode 100644 index 000000000..0268b75d3 --- /dev/null +++ b/tests/framework/utils/eventually.go @@ -0,0 +1,93 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "testing" + "time" +) + +// eventuallyPollInterval is the cadence of Eventually. It is deliberately not +// a parameter: consumers poll cheap idempotent operations (kubectl applies, +// rgw admin / S3 reads), and uniform call sites beat another knob. Add a +// variant if a real need for a different cadence ever appears. +const eventuallyPollInterval = time.Second + +// Eventually polls cond on the calling goroutine every second until it +// returns nil, the timeout elapses, or ctx is canceled. Each failed attempt +// is logged via t.Logf so CI output shows liveness attributed to the right +// (sub)test, and the last attempt's error is wrapped into the returned error. +// +// The context passed to cond is derived from ctx with the overall timeout +// applied, so a cond that honors it stops at the deadline instead of +// overrunning it. To additionally bound each attempt, wrap cond with +// AttemptTimeout. +// +// cond must not call assert or require: report transient failures by +// returning an error, and make assertions on captured state after the wait. +// Eventually never runs cond on another goroutine — a hung attempt would keep +// executing concurrently with its retry, and require's FailNow is unsafe off +// the test goroutine (which is also why testify's assert.Eventually is not +// used here). +func Eventually(ctx context.Context, t *testing.T, timeout time.Duration, desc string, cond func(context.Context) error) error { + t.Helper() + + loopCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + for attempt := 1; ; attempt++ { + err := cond(loopCtx) + if err == nil { + return nil + } + t.Logf("waiting for %s (attempt %d): %v", desc, attempt, err) + + select { + case <-loopCtx.Done(): + if cerr := ctx.Err(); cerr != nil { + return fmt.Errorf("canceled while waiting for %s: %w (last error: %v)", desc, cerr, err) + } + return fmt.Errorf("condition not met within %s: %s: %w", timeout, desc, err) + case <-time.After(eventuallyPollInterval): + } + } +} + +// AttemptTimeout bounds each attempt of cond with its own deadline, for +// operations that can hang but would succeed if canceled and retried. The +// bound is cooperative: cond only stops early if it honors the context it is +// given (requests built with http.NewRequestWithContext, exec.CommandContext, +// SDK *WithContext calls, ...). A cond that ignores its context is unaffected +// by AttemptTimeout: +// +// // WRONG: http.Get ignores ctx, so nothing cancels a hung fetch +// utils.AttemptTimeout(10*time.Second, func(ctx context.Context) error { +// _, err := http.Get(url) +// return err +// }) +// +// Operations that cannot honor a context must be bounded at the operation +// level instead (e.g. http.Client{Timeout: ...}). +func AttemptTimeout(d time.Duration, cond func(context.Context) error) func(context.Context) error { + return func(ctx context.Context) error { + attemptCtx, cancel := context.WithTimeout(ctx, d) + defer cancel() + return cond(attemptCtx) + } +} diff --git a/tests/framework/utils/eventually_test.go b/tests/framework/utils/eventually_test.go new file mode 100644 index 000000000..ffaf7d8e6 --- /dev/null +++ b/tests/framework/utils/eventually_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEventuallySucceedsImmediately(t *testing.T) { + calls := 0 + err := Eventually(context.Background(), t, time.Minute, "immediate success", func(context.Context) error { + calls++ + return nil + }) + require.NoError(t, err) + assert.Equal(t, 1, calls) +} + +func TestEventuallyRetriesUntilSuccess(t *testing.T) { + calls := 0 + err := Eventually(context.Background(), t, time.Minute, "second attempt succeeds", func(context.Context) error { + calls++ + if calls < 2 { + return errors.New("not yet") + } + return nil + }) + require.NoError(t, err) + assert.Equal(t, 2, calls) +} + +func TestEventuallyTimeoutWrapsLastError(t *testing.T) { + cause := errors.New("still broken") + err := Eventually(context.Background(), t, 10*time.Millisecond, "never succeeds", func(context.Context) error { + return cause + }) + require.Error(t, err) + assert.ErrorIs(t, err, cause) + assert.Contains(t, err.Error(), "never succeeds") +} + +func TestEventuallyParentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := Eventually(ctx, t, time.Minute, "canceled parent", func(context.Context) error { + return errors.New("not yet") + }) + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestEventuallyCondSeesOverallDeadline(t *testing.T) { + err := Eventually(context.Background(), t, 10*time.Millisecond, "cond honors ctx", func(ctx context.Context) error { + // a cooperative operation blocks only until the loop deadline + <-ctx.Done() + return ctx.Err() + }) + require.Error(t, err) + assert.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestAttemptTimeoutCancelsHungAttempt(t *testing.T) { + calls := 0 + err := Eventually(context.Background(), t, time.Minute, "hung attempt is canceled and retried", + AttemptTimeout(10*time.Millisecond, func(ctx context.Context) error { + calls++ + if calls < 2 { + // simulate an operation that hangs until canceled + <-ctx.Done() + return ctx.Err() + } + return nil + })) + require.NoError(t, err) + assert.Equal(t, 2, calls) +} diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 06fa218ca..ffdb9f5c9 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -237,6 +237,26 @@ func (k8sh *K8sHelper) ResourceOperation(action string, manifest string) error { return fmt.Errorf("could Not create resource in args : %v -- %v", args, err) } +// applyRetryTimeout bounds ApplyWithRetry. A webhook backend (cert-manager, +// trust-manager, ...) is typically reachable within seconds of its deployment +// reporting ready; two minutes sits comfortably above that without hanging a +// genuinely broken setup for long. +const applyRetryTimeout = 2 * time.Minute + +// ApplyWithRetry retries "kubectl apply" of manifest until it succeeds or +// applyRetryTimeout elapses. It exists for resources guarded by an admission +// webhook whose backing service may not be reachable the instant its +// deployment reports ready (e.g. cert-manager / trust-manager): "helm --wait" +// returns once the webhook deployment is ready, but the first apply can still +// fail with "failed calling webhook ... connection refused". description names +// the resource in the retry log lines. See Eventually for polling semantics. +func (k8sh *K8sHelper) ApplyWithRetry(ctx context.Context, manifest, description string) error { + return Eventually(ctx, k8sh.T(), applyRetryTimeout, "apply "+description, + func(context.Context) error { + return k8sh.ResourceOperation("apply", manifest) + }) +} + // DeletePod performs a kubectl delete pod on the given pod func (k8sh *K8sHelper) DeletePod(namespace, name string) error { args := append([]string{"--grace-period=0", "pod"}, name) diff --git a/tests/integration/ceph_base_keystone_test.go b/tests/integration/ceph_base_keystone_test.go index b374f7f5d..bcb7f5dff 100644 --- a/tests/integration/ceph_base_keystone_test.go +++ b/tests/integration/ceph_base_keystone_test.go @@ -114,27 +114,27 @@ func InstallKeystoneInTestCluster(shelper *utils.K8sHelper, namespace string) er return err } - if err := shelper.ResourceOperation("apply", keystoneApiClusterIssuer(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, keystoneApiClusterIssuer(namespace), "ClusterIssuer"); err != nil { logger.Errorf("Could not apply ClusterIssuer in namespace %s: %s", namespace, err) return err } - if err := shelper.ResourceOperation("apply", keystoneApiCaCertificate(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, keystoneApiCaCertificate(namespace), "CA Certificate"); err != nil { logger.Errorf("Could not apply ClusterIssuer CA Certificate in namespace %s: %s", namespace, err) return err } - if err := shelper.ResourceOperation("apply", keystoneApiCaIssuer(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, keystoneApiCaIssuer(namespace), "CA Issuer"); err != nil { logger.Errorf("Could not install CA Issuer in namespace %s: %s", namespace, err) return err } - if err := shelper.ResourceOperation("apply", keystoneApiCertificate(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, keystoneApiCertificate(namespace), "Certificate"); err != nil { logger.Errorf("Could not create Certificate (request) in namespace %s", namespace) return err } - if err := shelper.ResourceOperation("apply", trustManagerBundle(namespace)); err != nil { + if err := shelper.ApplyWithRetry(ctx, trustManagerBundle(namespace), "trust-manager Bundle"); err != nil { logger.Errorf("Could not create CA Certificate Bundle in namespace %s", namespace) return err }