forked from rook/rook
Joshua Hoblitt
4fb04e638d
The keystone auth suite installs cert-manager and trust-manager with helm --wait and then immediately applies Issuer, Certificate, and Bundle resources. helm --wait returns when the webhook deployments report ready, but the webhook service endpoints may not be programmed on the apiserver's node yet, so the apply is rejected with 'failed calling webhook ... connect: connection refused' and the suite fails before any test runs. This is currently the dominant failure mode of the keystone suite, seen on master and on PRs that cannot have caused it (dependabot github-actions bumps, mergify backports). Add utils.Eventually, a timeout-based polling primitive intended as the shared replacement for hand-rolled retry loops in test code: - cond is func(ctx) error rather than func() bool, so each failed attempt logs its reason via t.Logf and the final timeout error wraps the last attempt's error. - cond runs on the calling goroutine and receives a context carrying the overall deadline, so cooperative operations stop at the deadline instead of overrunning it. It is never run on another goroutine: a hung attempt would keep executing concurrently with its retry, and require's FailNow is unsafe off the test goroutine. - utils.AttemptTimeout decorates a cond with a per-attempt deadline for operations that can hang but would succeed if canceled and retried. The bound is cooperative; conds that cannot honor a context must be bounded at the operation level instead. Expose it as a K8sHelper.ApplyWithRetry method that retries kubectl apply until the webhooks accept connections, and use it for the keystone setup applies instead of failing the suite on the first attempt. Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
97 lines
2.7 KiB
Go
97 lines
2.7 KiB
Go
/*
|
|
Copyright 2026 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package utils
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestEventuallySucceedsImmediately(t *testing.T) {
|
|
calls := 0
|
|
err := Eventually(context.Background(), t, time.Minute, "immediate success", func(context.Context) error {
|
|
calls++
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 1, calls)
|
|
}
|
|
|
|
func TestEventuallyRetriesUntilSuccess(t *testing.T) {
|
|
calls := 0
|
|
err := Eventually(context.Background(), t, time.Minute, "second attempt succeeds", func(context.Context) error {
|
|
calls++
|
|
if calls < 2 {
|
|
return errors.New("not yet")
|
|
}
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 2, calls)
|
|
}
|
|
|
|
func TestEventuallyTimeoutWrapsLastError(t *testing.T) {
|
|
cause := errors.New("still broken")
|
|
err := Eventually(context.Background(), t, 10*time.Millisecond, "never succeeds", func(context.Context) error {
|
|
return cause
|
|
})
|
|
require.Error(t, err)
|
|
assert.ErrorIs(t, err, cause)
|
|
assert.Contains(t, err.Error(), "never succeeds")
|
|
}
|
|
|
|
func TestEventuallyParentCancellation(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
err := Eventually(ctx, t, time.Minute, "canceled parent", func(context.Context) error {
|
|
return errors.New("not yet")
|
|
})
|
|
require.Error(t, err)
|
|
assert.ErrorIs(t, err, context.Canceled)
|
|
}
|
|
|
|
func TestEventuallyCondSeesOverallDeadline(t *testing.T) {
|
|
err := Eventually(context.Background(), t, 10*time.Millisecond, "cond honors ctx", func(ctx context.Context) error {
|
|
// a cooperative operation blocks only until the loop deadline
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
})
|
|
require.Error(t, err)
|
|
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
|
}
|
|
|
|
func TestAttemptTimeoutCancelsHungAttempt(t *testing.T) {
|
|
calls := 0
|
|
err := Eventually(context.Background(), t, time.Minute, "hung attempt is canceled and retried",
|
|
AttemptTimeout(10*time.Millisecond, func(ctx context.Context) error {
|
|
calls++
|
|
if calls < 2 {
|
|
// simulate an operation that hangs until canceled
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
}
|
|
return nil
|
|
}))
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 2, calls)
|
|
}
|