test: add the object suite test util toolkit

Add the shared helpers the object test packages build on:
- wait4: watch-based Assert/Require waiters (Create/Delete/Condition/Absent)
  that block on Kubernetes watch events rather than fixed-interval polling,
  plus Eventually for non-k8s state (rgw admin/S3/SNS), a pod-log waiter, and
  the readiness predicates used with them.
- secrets: verification of the Secret references CRDs publish in status.
- fixture: create-with-t.Cleanup helpers for pure-cleanup resources, and
  constructors for ObjectBucketClaim test resources.
- client: rgw admin and SNS client builders, S3 credentials/endpoint, and the
  store's TLS cert plus a verification-skipping HTTP client.

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
(cherry picked from commit b59101d8e8)
This commit is contained in:
Joshua Hoblitt authored and Mergify committed 2026-06-29 20:33:44 +00:00
1 parent d83ad67390
commit 29af430739
9 files changed
+1082

No files matched your search

@@ -0,0 +1,63 @@
/*
Copyright 2025 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package client builds the clients the object tests use to reach a
// CephObjectStore — the rgw admin and SNS clients and the S3 credentials and
// endpoint they need — along with the store's TLS certificate and a
// verification-skipping HTTP client for the TLS pass.
package client
import (
"context"
"net/http"
"github.com/ceph/go-ceph/rgw/admin"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
)
func NewAdminClient(objectStore *cephv1.CephObjectStore, installer *installer.CephInstaller, k8sh *utils.K8sHelper, tlsEnable bool) (*admin.API, error) {
accessKey, secretKey, err := GetS3Credentials(objectStore, installer)
if err != nil {
return nil, errors.Wrap(err, "failed to get s3 credentials")
}
endpoint, err := GetS3Endpoint(objectStore, k8sh, tlsEnable)
if err != nil {
return nil, errors.Wrap(err, "failed to get s3 endpoint")
}
httpClient := &http.Client{}
if tlsEnable {
httpClient = InsecureHTTPClient()
}
adminClient, err := admin.New(endpoint, accessKey, secretKey, httpClient)
if err != nil {
return nil, errors.Wrap(err, "failed to setup ceph admin client")
}
// verify that admin api is working
_, err = adminClient.GetInfo(context.Background())
if err != nil {
return nil, errors.Wrap(err, "admin client is not working")
}
return adminClient, nil
}
@@ -0,0 +1,75 @@
/*
Copyright 2025 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"context"
"encoding/json"
"fmt"
"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
)
func GetS3Credentials(objectStore *cephv1.CephObjectStore, installer *installer.CephInstaller) (string, string, error) {
output, err := installer.Execute("radosgw-admin", []string{"user", "info", "--uid=dashboard-admin", fmt.Sprintf("--rgw-realm=%s", objectStore.Name)}, objectStore.Namespace)
if err != nil {
return "", "", errors.Wrap(err, "failed to get user info")
}
// extract api creds from json output
var userInfo map[string]interface{}
err = json.Unmarshal([]byte(output), &userInfo)
if err != nil {
return "", "", errors.Wrap(err, "failed to unmarshal user info")
}
accessKey, ok := userInfo["keys"].([]interface{})[0].(map[string]interface{})["access_key"].(string)
if !ok {
return "", "", errors.New("failed to get access key")
}
secretKey, ok := userInfo["keys"].([]interface{})[0].(map[string]interface{})["secret_key"].(string)
if !ok {
return "", "", errors.New("failed to get secret key")
}
return accessKey, secretKey, nil
}
func GetS3Endpoint(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, tlsEnable bool) (string, error) {
ctx := context.TODO()
// extract rgw endpoint from k8s svc
svc, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{})
if err != nil {
return "", errors.Wrap(err, "failed to get objectstore svc")
}
schema, port := "http://", "80"
if tlsEnable {
schema, port = "https://", "443"
}
endpoint := schema + svc.Spec.ClusterIP + ":" + port
return endpoint, nil
}
@@ -0,0 +1,93 @@
/*
Copyright 2025 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"context"
"net/url"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/sns"
smithyendpoints "github.com/aws/smithy-go/endpoints"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
)
// based on the s3 endpoint example in the official docs:
// https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-endpoints.html
type snsResolverV2 struct {
endpoint string
}
func (r *snsResolverV2) ResolveEndpoint(ctx context.Context, params sns.EndpointParameters) (smithyendpoints.Endpoint, error) {
u, err := url.Parse(r.endpoint)
if err != nil {
return smithyendpoints.Endpoint{}, err
}
return smithyendpoints.Endpoint{
URI: *u,
}, nil
}
func NewSNSClient(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) {
ctx := context.TODO()
accessKey, secretKey, err := GetS3Credentials(objectStore, installer)
if err != nil {
return nil, errors.Wrap(err, "failed to get s3 credentials")
}
loadOpts := []func(*config.LoadOptions) error{
config.WithRegion("us-east-1"),
config.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(
accessKey,
secretKey,
"",
),
),
}
if tlsEnable {
loadOpts = append(loadOpts, config.WithHTTPClient(InsecureHTTPClient()))
}
cfg, err := config.LoadDefaultConfig(ctx, loadOpts...)
if err != nil {
return nil, errors.Wrap(err, "failed to load default aws config")
}
endpoint, err := GetS3Endpoint(objectStore, k8sh, tlsEnable)
if err != nil {
return nil, errors.Wrap(err, "failed to get sns endpoint")
}
snsClient := sns.NewFromConfig(cfg, func(o *sns.Options) {
o.EndpointResolverV2 = &snsResolverV2{endpoint: endpoint}
})
// sanity check that client is working
_, err = snsClient.ListTopics(ctx, &sns.ListTopicsInput{})
if err != nil {
return nil, errors.Wrap(err, "failed to sns client sanity check")
}
return snsClient, nil
}
@@ -0,0 +1,85 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"context"
cryptotls "crypto/tls"
"fmt"
"net/http"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rook/rook/tests/framework/utils"
)
// InsecureHTTPClient returns an http.Client that skips TLS verification, for
// reaching a test CephObjectStore served with a cluster-signed cert the test
// process does not trust.
func InsecureHTTPClient() *http.Client {
return &http.Client{
Transport: &http.Transport{
// nolint:gosec // skip TLS verification as this is a test
TLSClientConfig: &cryptotls.Config{InsecureSkipVerify: true},
},
}
}
// GenerateRgwTLSCertSecret generates a cluster-signed TLS certificate for
// rgwServiceName and stores it as an Opaque Secret named secretName in
// namespace. The secret holds the concatenated key, certificate, and CA bundle
// under the data key "cert", which is the format a CephObjectStore expects when
// referenced via Gateway.SSLCertificateRef.
func GenerateRgwTLSCertSecret(t *testing.T, k8sh *utils.K8sHelper, namespace, secretName, rgwServiceName string) {
ctx := context.TODO()
root, err := utils.FindRookRoot()
require.NoError(t, err, "failed to get rook root")
tlscertdir := t.TempDir()
cmdArgs := utils.CommandArgs{
Command: filepath.Join(root, "tests/scripts/generate-tls-config.sh"),
CmdArgs: []string{tlscertdir, rgwServiceName, namespace},
}
cmdOut := utils.ExecuteCommand(cmdArgs)
require.NoError(t, cmdOut.Err)
tlsKeyIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".key"))
require.NoError(t, err)
tlsCertIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".crt"))
require.NoError(t, err)
tlsCaCertIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".ca"))
require.NoError(t, err)
secretCertOut := fmt.Sprintf("%s%s%s", tlsKeyIn, tlsCertIn, tlsCaCertIn)
tlsK8sSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: namespace,
},
Data: map[string][]byte{
"cert": []byte(secretCertOut),
},
}
_, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(ctx, tlsK8sSecret, metav1.CreateOptions{})
require.NoError(t, err)
}
@@ -0,0 +1,74 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package fixture provides setup helpers for resources whose teardown is pure
// cleanup rather than part of what a test asserts. Each helper creates the
// resource and registers its deletion via t.Cleanup, so the resource is
// removed even when the test fails partway through and never reaches its
// trailing teardown steps — important because the tests share one cluster.
//
// Deletions that are themselves assertions (e.g. "deleting the OBC must not
// delete the user") do not belong here; keep those as ordered subtests.
package fixture
import (
"context"
"testing"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rook/rook/tests/framework/utils"
)
// The helpers manage their own contexts rather than taking one from the
// caller: cleanup runs after the test finishes, when a test-scoped context
// (e.g. t.Context()) has already been canceled.
// RequireNamespace creates ns and registers its deletion via t.Cleanup.
// Deletion is issued without waiting for it to complete; namespace teardown is
// asynchronous and nothing in these tests depends on it finishing.
func RequireNamespace(t *testing.T, k8sh *utils.K8sHelper, ns *corev1.Namespace) {
t.Helper()
_, err := k8sh.Clientset.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
require.NoError(t, err)
t.Cleanup(func() {
err := k8sh.Clientset.CoreV1().Namespaces().Delete(context.Background(), ns.Name, metav1.DeleteOptions{})
if err != nil && !kerrors.IsNotFound(err) {
t.Errorf("failed to delete namespace %q: %v", ns.Name, err)
}
})
}
// RequireStorageClass creates sc and registers its deletion via t.Cleanup.
func RequireStorageClass(t *testing.T, k8sh *utils.K8sHelper, sc *storagev1.StorageClass) {
t.Helper()
_, err := k8sh.Clientset.StorageV1().StorageClasses().Create(context.Background(), sc, metav1.CreateOptions{})
require.NoError(t, err)
t.Cleanup(func() {
err := k8sh.Clientset.StorageV1().StorageClasses().Delete(context.Background(), sc.Name, metav1.DeleteOptions{})
if err != nil && !kerrors.IsNotFound(err) {
t.Errorf("failed to delete storageclass %q: %v", sc.Name, err)
}
})
}
@@ -0,0 +1,39 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fixture
import (
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
)
// StorageClass returns a StorageClass for the lib-bucket-provisioner bucket
// provisioner backed by objectStore, for use by ObjectBucketClaims.
func StorageClass(name string, objectStore *cephv1.CephObjectStore) *storagev1.StorageClass {
return &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
Parameters: map[string]string{
"objectStoreName": objectStore.Name,
"objectStoreNamespace": objectStore.Namespace,
},
}
}
@@ -0,0 +1,94 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package secrets holds verification helpers for the Secret references that
// object CRDs publish in their status, shared by more than one object test
// package. Verification helpers for a different resource belong in a package
// named for that resource, not here.
package secrets
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/utils"
"github.com/rook/rook/tests/integration/object/util/wait4"
)
// RequireStatusRefs runs a subtest named desc that waits for the status secret
// references returned by refsOf to reach len(expected) — the controller
// repopulates them asynchronously after a reconcile — and then asserts each ref
// against its live secret via AssertRefs. refsOf must tolerate an unpopulated
// status (return nil).
func RequireStatusRefs[T, L runtime.Object](
t *testing.T,
k8sh *utils.K8sHelper,
client wait4.NamespacedWatcher[T, L],
name, desc string,
refsOf func(T) []cephv1.SecretReference,
expected ...*corev1.Secret,
) {
t.Helper()
t.Run(desc, func(t *testing.T) {
ctx := t.Context()
live := wait4.RequireCondition(ctx, t, client, name, func(obj T) bool {
return len(refsOf(obj)) == len(expected)
}, wait4.TimeoutShort)
AssertRefs(ctx, t, k8sh, refsOf(live), expected...)
})
}
// AssertRefs verifies that refs contains exactly one SecretReference per
// expected Secret and that each matches the live secret's Name, Namespace,
// UID, and ResourceVersion. Live secrets are fetched from each expected
// secret's own namespace.
func AssertRefs(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, refs []cephv1.SecretReference, expected ...*corev1.Secret) {
t.Helper()
assert.Len(t, refs, len(expected))
for _, secret := range expected {
secretRef, err := findSecretRef(refs, secret.Name)
require.NoError(t, err)
// fetch the live secret for UID and ResourceVersion
liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
require.NoError(t, err)
assert.Equal(t, liveSecret.Name, secretRef.Name)
assert.Equal(t, liveSecret.Namespace, secretRef.Namespace)
assert.Equal(t, liveSecret.UID, secretRef.UID)
assert.Equal(t, liveSecret.ResourceVersion, secretRef.ResourceVersion)
}
}
// findSecretRef returns the SecretReference for the named secret.
func findSecretRef(refs []cephv1.SecretReference, name string) (cephv1.SecretReference, error) {
for _, ref := range refs {
if ref.Name == name {
return ref, nil
}
}
return cephv1.SecretReference{}, fmt.Errorf("secretReference for secret %q not found", name)
}
@@ -0,0 +1,71 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package wait4
import (
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
)
// ObjectStore reports whether a CephObjectStore has been reconciled to Ready.
func ObjectStore(os *cephv1.CephObjectStore) bool {
return os.Status != nil && os.Status.Phase == cephv1.ConditionReady
}
// ObjectStoreUser reports whether a CephObjectStoreUser has been reconciled to
// Ready.
func ObjectStoreUser(u *cephv1.CephObjectStoreUser) bool {
return ObjectStoreUserPhase(string(cephv1.ConditionReady))(u)
}
// ObjectStoreUserPhase returns a predicate reporting whether a
// CephObjectStoreUser's .status.phase matches phase. The parameter is a plain
// string because the cephv1 phase constants are a mix of ConditionType and
// ConditionReason (e.g. string(cephv1.ReconcileFailed)).
func ObjectStoreUserPhase(phase string) func(*cephv1.CephObjectStoreUser) bool {
return func(u *cephv1.CephObjectStoreUser) bool {
return u.Status != nil && u.Status.Phase == phase
}
}
// BucketTopic reports whether a CephBucketTopic has been reconciled to Ready
// with its ARN populated. Note that .status.ARN is nil while .status.phase ==
// Reconciling, so the phase check alone is not sufficient.
func BucketTopic(bt *cephv1.CephBucketTopic) bool {
return BucketTopicPhase(string(cephv1.ConditionReady))(bt) && bt.Status.ARN != nil
}
// BucketTopicPhase returns a predicate reporting whether a CephBucketTopic's
// .status.phase matches phase. The parameter is a plain string because the
// cephv1 phase constants are a mix of ConditionType and ConditionReason (e.g.
// string(cephv1.ReconcileFailed)).
func BucketTopicPhase(phase string) func(*cephv1.CephBucketTopic) bool {
return func(bt *cephv1.CephBucketTopic) bool {
return bt.Status != nil && bt.Status.Phase == phase
}
}
// OBCBound reports whether an ObjectBucketClaim has reached the Bound phase.
func OBCBound(obc *bktv1alpha1.ObjectBucketClaim) bool {
return obc.Status.Phase == bktv1alpha1.ObjectBucketClaimStatusPhaseBound
}
// OBBound reports whether an ObjectBucket has reached the Bound phase.
func OBBound(ob *bktv1alpha1.ObjectBucket) bool {
return ob.Status.Phase == bktv1alpha1.ObjectBucketStatusPhaseBound
}
@@ -0,0 +1,488 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package wait4
import (
"bufio"
"context"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"github.com/rook/rook/tests/framework/utils"
)
// Default timeouts for waiting on operator reconciliation in the object
// tests. Bespoke waits (e.g. the shared store's multi-minute teardown ladder)
// still pass explicit durations.
const (
// TimeoutShort covers routine status changes on an already-running
// operator and rgw: condition waits, deletions, field updates.
TimeoutShort = 40 * time.Second
// TimeoutMedium covers resource creation that needs a full reconcile,
// such as a CephBucketTopic becoming Ready with its ARN set.
TimeoutMedium = time.Minute
// TimeoutLong covers first-reconcile waits that may race rgw startup,
// such as CephObjectStoreUser creation right after the store comes up.
TimeoutLong = 2 * time.Minute
)
// NamespacedWatcher is satisfied by any typed namespaced client (corev1
// ServiceInterface, cephv1 CephObjectStoreUserInterface, etc.). T is the
// resource's pointer type and L its list type, both inferred at the call site
// from the client's Get/List return types.
type NamespacedWatcher[T, L runtime.Object] interface {
Get(ctx context.Context, name string, opts metav1.GetOptions) (T, error)
List(ctx context.Context, opts metav1.ListOptions) (L, error)
Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
}
// nameScopedListWatch returns a ListWatch restricted to a single resource name.
func nameScopedListWatch[T, L runtime.Object](ctx context.Context, client NamespacedWatcher[T, L], name string) *cache.ListWatch {
fs := fields.OneTermEqualSelector("metadata.name", name).String()
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
opts.FieldSelector = fs
return client.List(ctx, opts)
},
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
opts.FieldSelector = fs
return client.Watch(ctx, opts)
},
}
}
// heartbeatInterval is how often a watch-based wait logs progress while it is
// still blocked.
const heartbeatInterval = 15 * time.Second
// heartbeat logs a periodic progress line for a watch-based wait that would
// otherwise block silently, so a slow reconcile shows up in CI output instead
// of looking like a hang. It logs nothing for waits shorter than
// heartbeatInterval; the caller must invoke the returned stop func when the
// wait returns.
func heartbeat(t *testing.T, desc string) (stop func()) {
t.Helper()
done := make(chan struct{})
go func() {
ticker := time.NewTicker(heartbeatInterval)
defer ticker.Stop()
for elapsed := heartbeatInterval; ; elapsed += heartbeatInterval {
select {
case <-done:
return
case <-ticker.C:
t.Logf("still waiting for %s (%s)", desc, elapsed)
}
}
}()
return func() { close(done) }
}
// waitForAbsent blocks until the named resource is gone or the timeout elapses.
//
// The wait is race-free against fast deletions: UntilWithSync performs an
// initial List (scoped to the resource name via a field selector) before
// starting the watch, and the precondition treats an empty store as "already
// gone". Subsequent deletion events are guaranteed to be delivered because the
// watch resumes from the List's resourceVersion.
func waitForAbsent[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedWatcher[T, L], name string, timeout time.Duration) error {
var objType T
lw := nameScopedListWatch(ctx, client, name)
waitCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
precondition := func(store cache.Store) (bool, error) {
// lw is name-scoped, so an empty store unambiguously means the object
// was already gone by the time the reflector's initial list completed.
return len(store.List()) == 0, nil
}
condition := func(ev watch.Event) (bool, error) {
return ev.Type == watch.Deleted, nil
}
defer heartbeat(t, fmt.Sprintf("%T %q to be removed", objType, name))()
if _, err := watchtools.UntilWithSync(waitCtx, lw, objType, precondition, condition); err != nil {
return fmt.Errorf("%T %q is still present: %w", objType, name, err)
}
return nil
}
// waitForCondition blocks until the named resource satisfies cond or the timeout
// elapses, returning the matching object. The initial List makes it race-free in
// the same way as waitForAbsent: a resource that already satisfies cond is
// detected immediately, and subsequent updates arrive as watch events.
func waitForCondition[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedWatcher[T, L], name string, cond func(T) bool, timeout time.Duration) (T, error) {
var objType T
var match T
lw := nameScopedListWatch(ctx, client, name)
waitCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
precondition := func(store cache.Store) (bool, error) {
for _, o := range store.List() {
if typed, ok := o.(T); ok && cond(typed) {
match = typed
return true, nil
}
}
return false, nil
}
condition := func(ev watch.Event) (bool, error) {
switch ev.Type {
case watch.Added, watch.Modified:
if typed, ok := ev.Object.(T); ok && cond(typed) {
match = typed
return true, nil
}
case watch.Deleted:
return false, fmt.Errorf("%T %q was deleted while waiting for condition", objType, name)
}
return false, nil
}
defer heartbeat(t, fmt.Sprintf("%T %q to be ready", objType, name))()
if _, err := watchtools.UntilWithSync(waitCtx, lw, objType, precondition, condition); err != nil {
return match, fmt.Errorf("%T %q did not satisfy condition: %w", objType, name, err)
}
return match, nil
}
// NamespacedDeleter is a NamespacedWatcher that can also delete by name.
type NamespacedDeleter[T, L runtime.Object] interface {
NamespacedWatcher[T, L]
Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
}
func deleteAndWait[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedDeleter[T, L], name string, timeout time.Duration) error {
var objType T
if err := client.Delete(ctx, name, metav1.DeleteOptions{}); err != nil {
return fmt.Errorf("failed to delete %T %q: %w", objType, name, err)
}
return waitForAbsent[T, L](ctx, t, client, name, timeout)
}
// AssertDelete deletes a single named resource and blocks until it is gone,
// reporting failures (including timeout) via assert.Fail (non-fatal). It returns
// true on success. Prefer this for teardown/cleanup, so a stuck finalizer on one
// resource does not prevent cleanup of the rest; use RequireDelete when later
// steps depend on the deletion having completed.
func AssertDelete[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedDeleter[T, L],
name string,
timeout time.Duration,
msgAndArgs ...interface{},
) bool {
t.Helper()
if err := deleteAndWait(ctx, t, client, name, timeout); err != nil {
return assert.Fail(t, err.Error(), msgAndArgs...)
}
return true
}
// RequireDelete is the fatal counterpart of AssertDelete: it stops the test
// (FailNow) if the resource cannot be deleted or is not gone before the timeout.
func RequireDelete[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedDeleter[T, L],
name string,
timeout time.Duration,
msgAndArgs ...interface{},
) {
t.Helper()
require.NoError(t, deleteAndWait(ctx, t, client, name, timeout), msgAndArgs...)
}
// AssertAbsent waits for a named resource to disappear without deleting it
// itself, reporting failures via assert.Fail (non-fatal). Use it for resources
// removed as a side effect of another action — e.g. an ObjectBucket that is
// garbage-collected when its ObjectBucketClaim is deleted. To delete and wait,
// use AssertDelete/RequireDelete.
func AssertAbsent[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedWatcher[T, L],
name string,
timeout time.Duration,
msgAndArgs ...interface{},
) bool {
t.Helper()
if err := waitForAbsent(ctx, t, client, name, timeout); err != nil {
return assert.Fail(t, err.Error(), msgAndArgs...)
}
return true
}
// RequireAbsent is the fatal counterpart of AssertAbsent.
func RequireAbsent[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedWatcher[T, L],
name string,
timeout time.Duration,
msgAndArgs ...interface{},
) {
t.Helper()
require.NoError(t, waitForAbsent(ctx, t, client, name, timeout), msgAndArgs...)
}
// NamespacedReadyWaiter is a NamespacedWatcher that can also create.
type NamespacedReadyWaiter[T, L runtime.Object] interface {
NamespacedWatcher[T, L]
Create(ctx context.Context, obj T, opts metav1.CreateOptions) (T, error)
}
// createAndWaitReady creates obj and, when ready is non-nil, blocks until ready
// reports true for the live object or the timeout elapses, returning the live
// object. Pass a nil ready predicate for resources that are usable as soon as
// they are created (Namespaces, Services, Secrets, ...).
//
// Unlike deletion, "ready" has no universal representation across resource
// types, so the readiness check is supplied by the caller.
func createAndWaitReady[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedReadyWaiter[T, L],
obj T,
ready func(T) bool,
timeout time.Duration,
) (T, error) {
var zero T
created, err := client.Create(ctx, obj, metav1.CreateOptions{})
if err != nil {
return zero, fmt.Errorf("failed to create %T: %w", obj, err)
}
if ready == nil {
return created, nil
}
accessor, err := meta.Accessor(created)
if err != nil {
return zero, fmt.Errorf("failed to read metadata of %T: %w", created, err)
}
return waitForCondition[T, L](ctx, t, client, accessor.GetName(), ready, timeout)
}
// AssertCreate creates obj and waits for the ready predicate, reporting failures
// (including timeout) via assert.Fail (non-fatal) like AssertDelete. It returns
// the live object and true on success. Prefer this for cleanup/best-effort
// creation where the test should continue on failure; for setup preconditions
// use RequireCreate.
func AssertCreate[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedReadyWaiter[T, L],
obj T,
ready func(T) bool,
timeout time.Duration,
msgAndArgs ...interface{},
) (T, bool) {
t.Helper()
live, err := createAndWaitReady(ctx, t, client, obj, ready, timeout)
if err != nil {
var zero T
assert.Fail(t, err.Error(), msgAndArgs...)
return zero, false
}
return live, true
}
// RequireCreate is the fatal counterpart of AssertCreate: it stops the test
// (FailNow) if the resource cannot be created or does not become ready, and
// returns the live object. This is the common case for setup preconditions, so
// the call site needs no follow-up require.True.
func RequireCreate[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedReadyWaiter[T, L],
obj T,
ready func(T) bool,
timeout time.Duration,
msgAndArgs ...interface{},
) T {
t.Helper()
live, err := createAndWaitReady(ctx, t, client, obj, ready, timeout)
require.NoError(t, err, msgAndArgs...)
return live
}
// AssertCondition waits for the named resource to satisfy cond, reporting
// failures (including timeout) via assert.Fail (non-fatal). It returns the
// matching object and true on success. Use it to wait on an already-created
// resource — e.g. an OBC reaching Bound, or a status field changing after an
// update; for the create-and-wait case use AssertCreate/RequireCreate.
func AssertCondition[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedWatcher[T, L],
name string,
cond func(T) bool,
timeout time.Duration,
msgAndArgs ...interface{},
) (T, bool) {
t.Helper()
match, err := waitForCondition(ctx, t, client, name, cond, timeout)
if err != nil {
var zero T
assert.Fail(t, err.Error(), msgAndArgs...)
return zero, false
}
return match, true
}
// RequireCondition is the fatal counterpart of AssertCondition.
func RequireCondition[T, L runtime.Object](
ctx context.Context,
t *testing.T,
client NamespacedWatcher[T, L],
name string,
cond func(T) bool,
timeout time.Duration,
msgAndArgs ...interface{},
) T {
t.Helper()
match, err := waitForCondition(ctx, t, client, name, cond, timeout)
require.NoError(t, err, msgAndArgs...)
return match
}
// AssertEventually polls cond until it returns nil or the timeout elapses,
// reporting failure via assert.Fail (non-fatal). It is for state the
// Kubernetes API cannot watch (the rgw admin API, S3, SNS); k8s resources
// should use the watch-based helpers above. The poll loop is utils.Eventually;
// this adds the assert-flavored, non-fatal reporting that pairs with the
// AssertCreate/AssertDelete/AssertCondition family.
//
// cond must not call assert or require: report transient failures by returning
// an error (which is surfaced in the timeout message), and make assertions on
// captured state after the wait. cond receives a context bounded by the wait
// deadline; thread it into the reads it performs.
func AssertEventually(ctx context.Context, t *testing.T, timeout time.Duration, desc string, cond func(context.Context) error) bool {
t.Helper()
if err := utils.Eventually(ctx, t, timeout, desc, cond); err != nil {
return assert.Fail(t, err.Error())
}
return true
}
// RequireEventually is the fatal counterpart of AssertEventually: it stops
// the test (FailNow) if cond does not succeed before the timeout. Use it
// when subsequent statements consume the polled result.
func RequireEventually(ctx context.Context, t *testing.T, timeout time.Duration, desc string, cond func(context.Context) error) {
t.Helper()
require.NoError(t, utils.Eventually(ctx, t, timeout, desc, cond))
}
// matchPodLog follows the logs of the first pod matching selector until match
// returns true for a line, the stream ends, or the wait deadline passes.
// Reaching the end of the stream without a match is an error: following the
// stream (rather than scanning a snapshot) is what lets the wait distinguish
// "absent" from "not logged yet".
func matchPodLog(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, selector labels.Selector, timeout time.Duration, match func(line string) bool) error {
t.Helper()
waitCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(waitCtx, metav1.ListOptions{LabelSelector: selector.String()})
if err != nil {
return fmt.Errorf("failed to list pods: %w", err)
}
if len(pods.Items) == 0 {
return fmt.Errorf("no pods found with labels %v in namespace %q", selector, namespace)
}
// if there are multiple pods, just pick the first one
pod := pods.Items[0]
t.Logf("matching against logs of pod %q (first match for labels %v)", pod.Name, selector)
req := k8sh.Clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Follow: true})
logStream, err := req.Stream(waitCtx)
if err != nil {
return fmt.Errorf("failed to stream logs from pod %q: %w", pod.Name, err)
}
defer logStream.Close()
scanner := bufio.NewScanner(logStream)
for scanner.Scan() {
if match(scanner.Text()) {
return nil
}
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("no matching log line from pod %q: %w", pod.Name, err)
}
return fmt.Errorf("log stream from pod %q ended without a matching line", pod.Name)
}
// AssertPodLog follows the logs of the first pod matching selector and reports
// failure via assert.Fail (non-fatal) if no line satisfies match before the
// timeout. It is for log-based state the Kubernetes API cannot watch (e.g. an
// operator logging a reconcile error); for a setup precondition use
// RequirePodLog.
func AssertPodLog(
ctx context.Context,
t *testing.T,
k8sh *utils.K8sHelper,
namespace string,
selector labels.Selector,
timeout time.Duration,
match func(line string) bool,
msgAndArgs ...interface{},
) bool {
t.Helper()
if err := matchPodLog(ctx, t, k8sh, namespace, selector, timeout, match); err != nil {
return assert.Fail(t, err.Error(), msgAndArgs...)
}
return true
}
// RequirePodLog is the fatal counterpart of AssertPodLog.
func RequirePodLog(
ctx context.Context,
t *testing.T,
k8sh *utils.K8sHelper,
namespace string,
selector labels.Selector,
timeout time.Duration,
match func(line string) bool,
msgAndArgs ...interface{},
) {
t.Helper()
require.NoError(t, matchPodLog(ctx, t, k8sh, namespace, selector, timeout, match), msgAndArgs...)
}