From 29af4307394d9f6eaefb03a9389486023aa116a2 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Fri, 26 Jun 2026 15:38:02 -0700 Subject: [PATCH] test: add the object suite test util toolkit Add the shared helpers the object test packages build on: - wait4: watch-based Assert/Require waiters (Create/Delete/Condition/Absent) that block on Kubernetes watch events rather than fixed-interval polling, plus Eventually for non-k8s state (rgw admin/S3/SNS), a pod-log waiter, and the readiness predicates used with them. - secrets: verification of the Secret references CRDs publish in status. - fixture: create-with-t.Cleanup helpers for pure-cleanup resources, and constructors for ObjectBucketClaim test resources. - client: rgw admin and SNS client builders, S3 credentials/endpoint, and the store's TLS cert plus a verification-skipping HTTP client. Signed-off-by: Joshua Hoblitt (cherry picked from commit b59101d8e8ee5b3a0e4c69624f205699dc7e885a) --- tests/integration/object/util/client/admin.go | 63 +++ tests/integration/object/util/client/s3.go | 75 +++ tests/integration/object/util/client/sns.go | 93 ++++ tests/integration/object/util/client/tls.go | 85 +++ .../object/util/fixture/fixture.go | 74 +++ tests/integration/object/util/fixture/obc.go | 39 ++ .../object/util/secrets/secrets.go | 94 ++++ tests/integration/object/util/wait4/ready.go | 71 +++ tests/integration/object/util/wait4/wait4.go | 488 ++++++++++++++++++ 9 files changed, 1082 insertions(+) create mode 100644 tests/integration/object/util/client/admin.go create mode 100644 tests/integration/object/util/client/s3.go create mode 100644 tests/integration/object/util/client/sns.go create mode 100644 tests/integration/object/util/client/tls.go create mode 100644 tests/integration/object/util/fixture/fixture.go create mode 100644 tests/integration/object/util/fixture/obc.go create mode 100644 tests/integration/object/util/secrets/secrets.go create mode 100644 tests/integration/object/util/wait4/ready.go create mode 100644 tests/integration/object/util/wait4/wait4.go diff --git a/tests/integration/object/util/client/admin.go b/tests/integration/object/util/client/admin.go new file mode 100644 index 000000000..73e22c36b --- /dev/null +++ b/tests/integration/object/util/client/admin.go @@ -0,0 +1,63 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package client builds the clients the object tests use to reach a +// CephObjectStore — the rgw admin and SNS clients and the S3 credentials and +// endpoint they need — along with the store's TLS certificate and a +// verification-skipping HTTP client for the TLS pass. +package client + +import ( + "context" + "net/http" + + "github.com/ceph/go-ceph/rgw/admin" + "github.com/pkg/errors" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" +) + +func NewAdminClient(objectStore *cephv1.CephObjectStore, installer *installer.CephInstaller, k8sh *utils.K8sHelper, tlsEnable bool) (*admin.API, error) { + accessKey, secretKey, err := GetS3Credentials(objectStore, installer) + if err != nil { + return nil, errors.Wrap(err, "failed to get s3 credentials") + } + + endpoint, err := GetS3Endpoint(objectStore, k8sh, tlsEnable) + if err != nil { + return nil, errors.Wrap(err, "failed to get s3 endpoint") + } + + httpClient := &http.Client{} + if tlsEnable { + httpClient = InsecureHTTPClient() + } + + adminClient, err := admin.New(endpoint, accessKey, secretKey, httpClient) + if err != nil { + return nil, errors.Wrap(err, "failed to setup ceph admin client") + } + + // verify that admin api is working + _, err = adminClient.GetInfo(context.Background()) + if err != nil { + return nil, errors.Wrap(err, "admin client is not working") + } + + return adminClient, nil +} diff --git a/tests/integration/object/util/client/s3.go b/tests/integration/object/util/client/s3.go new file mode 100644 index 000000000..f521c9eed --- /dev/null +++ b/tests/integration/object/util/client/s3.go @@ -0,0 +1,75 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" +) + +func GetS3Credentials(objectStore *cephv1.CephObjectStore, installer *installer.CephInstaller) (string, string, error) { + output, err := installer.Execute("radosgw-admin", []string{"user", "info", "--uid=dashboard-admin", fmt.Sprintf("--rgw-realm=%s", objectStore.Name)}, objectStore.Namespace) + if err != nil { + return "", "", errors.Wrap(err, "failed to get user info") + } + + // extract api creds from json output + var userInfo map[string]interface{} + err = json.Unmarshal([]byte(output), &userInfo) + if err != nil { + return "", "", errors.Wrap(err, "failed to unmarshal user info") + } + + accessKey, ok := userInfo["keys"].([]interface{})[0].(map[string]interface{})["access_key"].(string) + if !ok { + return "", "", errors.New("failed to get access key") + } + + secretKey, ok := userInfo["keys"].([]interface{})[0].(map[string]interface{})["secret_key"].(string) + if !ok { + return "", "", errors.New("failed to get secret key") + } + + return accessKey, secretKey, nil +} + +func GetS3Endpoint(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, tlsEnable bool) (string, error) { + ctx := context.TODO() + + // extract rgw endpoint from k8s svc + svc, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{}) + if err != nil { + return "", errors.Wrap(err, "failed to get objectstore svc") + } + + schema, port := "http://", "80" + if tlsEnable { + schema, port = "https://", "443" + } + + endpoint := schema + svc.Spec.ClusterIP + ":" + port + + return endpoint, nil +} diff --git a/tests/integration/object/util/client/sns.go b/tests/integration/object/util/client/sns.go new file mode 100644 index 000000000..cd9a1da97 --- /dev/null +++ b/tests/integration/object/util/client/sns.go @@ -0,0 +1,93 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "net/url" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/sns" + smithyendpoints "github.com/aws/smithy-go/endpoints" + "github.com/pkg/errors" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" +) + +// based on the s3 endpoint example in the official docs: +// https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-endpoints.html +type snsResolverV2 struct { + endpoint string +} + +func (r *snsResolverV2) ResolveEndpoint(ctx context.Context, params sns.EndpointParameters) (smithyendpoints.Endpoint, error) { + u, err := url.Parse(r.endpoint) + if err != nil { + return smithyendpoints.Endpoint{}, err + } + return smithyendpoints.Endpoint{ + URI: *u, + }, nil +} + +func NewSNSClient(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) { + ctx := context.TODO() + + accessKey, secretKey, err := GetS3Credentials(objectStore, installer) + if err != nil { + return nil, errors.Wrap(err, "failed to get s3 credentials") + } + + loadOpts := []func(*config.LoadOptions) error{ + config.WithRegion("us-east-1"), + config.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider( + accessKey, + secretKey, + "", + ), + ), + } + if tlsEnable { + loadOpts = append(loadOpts, config.WithHTTPClient(InsecureHTTPClient())) + } + + cfg, err := config.LoadDefaultConfig(ctx, loadOpts...) + if err != nil { + return nil, errors.Wrap(err, "failed to load default aws config") + } + + endpoint, err := GetS3Endpoint(objectStore, k8sh, tlsEnable) + if err != nil { + return nil, errors.Wrap(err, "failed to get sns endpoint") + } + + snsClient := sns.NewFromConfig(cfg, func(o *sns.Options) { + o.EndpointResolverV2 = &snsResolverV2{endpoint: endpoint} + }) + + // sanity check that client is working + _, err = snsClient.ListTopics(ctx, &sns.ListTopicsInput{}) + if err != nil { + return nil, errors.Wrap(err, "failed to sns client sanity check") + } + + return snsClient, nil +} diff --git a/tests/integration/object/util/client/tls.go b/tests/integration/object/util/client/tls.go new file mode 100644 index 000000000..8646c7763 --- /dev/null +++ b/tests/integration/object/util/client/tls.go @@ -0,0 +1,85 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + cryptotls "crypto/tls" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rook/rook/tests/framework/utils" +) + +// InsecureHTTPClient returns an http.Client that skips TLS verification, for +// reaching a test CephObjectStore served with a cluster-signed cert the test +// process does not trust. +func InsecureHTTPClient() *http.Client { + return &http.Client{ + Transport: &http.Transport{ + // nolint:gosec // skip TLS verification as this is a test + TLSClientConfig: &cryptotls.Config{InsecureSkipVerify: true}, + }, + } +} + +// GenerateRgwTLSCertSecret generates a cluster-signed TLS certificate for +// rgwServiceName and stores it as an Opaque Secret named secretName in +// namespace. The secret holds the concatenated key, certificate, and CA bundle +// under the data key "cert", which is the format a CephObjectStore expects when +// referenced via Gateway.SSLCertificateRef. +func GenerateRgwTLSCertSecret(t *testing.T, k8sh *utils.K8sHelper, namespace, secretName, rgwServiceName string) { + ctx := context.TODO() + + root, err := utils.FindRookRoot() + require.NoError(t, err, "failed to get rook root") + + tlscertdir := t.TempDir() + cmdArgs := utils.CommandArgs{ + Command: filepath.Join(root, "tests/scripts/generate-tls-config.sh"), + CmdArgs: []string{tlscertdir, rgwServiceName, namespace}, + } + cmdOut := utils.ExecuteCommand(cmdArgs) + require.NoError(t, cmdOut.Err) + + tlsKeyIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".key")) + require.NoError(t, err) + tlsCertIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".crt")) + require.NoError(t, err) + tlsCaCertIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".ca")) + require.NoError(t, err) + + secretCertOut := fmt.Sprintf("%s%s%s", tlsKeyIn, tlsCertIn, tlsCaCertIn) + tlsK8sSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Data: map[string][]byte{ + "cert": []byte(secretCertOut), + }, + } + _, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(ctx, tlsK8sSecret, metav1.CreateOptions{}) + require.NoError(t, err) +} diff --git a/tests/integration/object/util/fixture/fixture.go b/tests/integration/object/util/fixture/fixture.go new file mode 100644 index 000000000..7897001a1 --- /dev/null +++ b/tests/integration/object/util/fixture/fixture.go @@ -0,0 +1,74 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package fixture provides setup helpers for resources whose teardown is pure +// cleanup rather than part of what a test asserts. Each helper creates the +// resource and registers its deletion via t.Cleanup, so the resource is +// removed even when the test fails partway through and never reaches its +// trailing teardown steps — important because the tests share one cluster. +// +// Deletions that are themselves assertions (e.g. "deleting the OBC must not +// delete the user") do not belong here; keep those as ordered subtests. +package fixture + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rook/rook/tests/framework/utils" +) + +// The helpers manage their own contexts rather than taking one from the +// caller: cleanup runs after the test finishes, when a test-scoped context +// (e.g. t.Context()) has already been canceled. + +// RequireNamespace creates ns and registers its deletion via t.Cleanup. +// Deletion is issued without waiting for it to complete; namespace teardown is +// asynchronous and nothing in these tests depends on it finishing. +func RequireNamespace(t *testing.T, k8sh *utils.K8sHelper, ns *corev1.Namespace) { + t.Helper() + + _, err := k8sh.Clientset.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Cleanup(func() { + err := k8sh.Clientset.CoreV1().Namespaces().Delete(context.Background(), ns.Name, metav1.DeleteOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + t.Errorf("failed to delete namespace %q: %v", ns.Name, err) + } + }) +} + +// RequireStorageClass creates sc and registers its deletion via t.Cleanup. +func RequireStorageClass(t *testing.T, k8sh *utils.K8sHelper, sc *storagev1.StorageClass) { + t.Helper() + + _, err := k8sh.Clientset.StorageV1().StorageClasses().Create(context.Background(), sc, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Cleanup(func() { + err := k8sh.Clientset.StorageV1().StorageClasses().Delete(context.Background(), sc.Name, metav1.DeleteOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + t.Errorf("failed to delete storageclass %q: %v", sc.Name, err) + } + }) +} diff --git a/tests/integration/object/util/fixture/obc.go b/tests/integration/object/util/fixture/obc.go new file mode 100644 index 000000000..4f06bb348 --- /dev/null +++ b/tests/integration/object/util/fixture/obc.go @@ -0,0 +1,39 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fixture + +import ( + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" +) + +// StorageClass returns a StorageClass for the lib-bucket-provisioner bucket +// provisioner backed by objectStore, for use by ObjectBucketClaims. +func StorageClass(name string, objectStore *cephv1.CephObjectStore) *storagev1.StorageClass { + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket", + Parameters: map[string]string{ + "objectStoreName": objectStore.Name, + "objectStoreNamespace": objectStore.Namespace, + }, + } +} diff --git a/tests/integration/object/util/secrets/secrets.go b/tests/integration/object/util/secrets/secrets.go new file mode 100644 index 000000000..b0fd423d4 --- /dev/null +++ b/tests/integration/object/util/secrets/secrets.go @@ -0,0 +1,94 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package secrets holds verification helpers for the Secret references that +// object CRDs publish in their status, shared by more than one object test +// package. Verification helpers for a different resource belong in a package +// named for that resource, not here. +package secrets + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/tests/framework/utils" + "github.com/rook/rook/tests/integration/object/util/wait4" +) + +// RequireStatusRefs runs a subtest named desc that waits for the status secret +// references returned by refsOf to reach len(expected) — the controller +// repopulates them asynchronously after a reconcile — and then asserts each ref +// against its live secret via AssertRefs. refsOf must tolerate an unpopulated +// status (return nil). +func RequireStatusRefs[T, L runtime.Object]( + t *testing.T, + k8sh *utils.K8sHelper, + client wait4.NamespacedWatcher[T, L], + name, desc string, + refsOf func(T) []cephv1.SecretReference, + expected ...*corev1.Secret, +) { + t.Helper() + t.Run(desc, func(t *testing.T) { + ctx := t.Context() + live := wait4.RequireCondition(ctx, t, client, name, func(obj T) bool { + return len(refsOf(obj)) == len(expected) + }, wait4.TimeoutShort) + AssertRefs(ctx, t, k8sh, refsOf(live), expected...) + }) +} + +// AssertRefs verifies that refs contains exactly one SecretReference per +// expected Secret and that each matches the live secret's Name, Namespace, +// UID, and ResourceVersion. Live secrets are fetched from each expected +// secret's own namespace. +func AssertRefs(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, refs []cephv1.SecretReference, expected ...*corev1.Secret) { + t.Helper() + + assert.Len(t, refs, len(expected)) + + for _, secret := range expected { + secretRef, err := findSecretRef(refs, secret.Name) + require.NoError(t, err) + + // fetch the live secret for UID and ResourceVersion + liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{}) + require.NoError(t, err) + + assert.Equal(t, liveSecret.Name, secretRef.Name) + assert.Equal(t, liveSecret.Namespace, secretRef.Namespace) + assert.Equal(t, liveSecret.UID, secretRef.UID) + assert.Equal(t, liveSecret.ResourceVersion, secretRef.ResourceVersion) + } +} + +// findSecretRef returns the SecretReference for the named secret. +func findSecretRef(refs []cephv1.SecretReference, name string) (cephv1.SecretReference, error) { + for _, ref := range refs { + if ref.Name == name { + return ref, nil + } + } + return cephv1.SecretReference{}, fmt.Errorf("secretReference for secret %q not found", name) +} diff --git a/tests/integration/object/util/wait4/ready.go b/tests/integration/object/util/wait4/ready.go new file mode 100644 index 000000000..2b4372465 --- /dev/null +++ b/tests/integration/object/util/wait4/ready.go @@ -0,0 +1,71 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wait4 + +import ( + bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" +) + +// ObjectStore reports whether a CephObjectStore has been reconciled to Ready. +func ObjectStore(os *cephv1.CephObjectStore) bool { + return os.Status != nil && os.Status.Phase == cephv1.ConditionReady +} + +// ObjectStoreUser reports whether a CephObjectStoreUser has been reconciled to +// Ready. +func ObjectStoreUser(u *cephv1.CephObjectStoreUser) bool { + return ObjectStoreUserPhase(string(cephv1.ConditionReady))(u) +} + +// ObjectStoreUserPhase returns a predicate reporting whether a +// CephObjectStoreUser's .status.phase matches phase. The parameter is a plain +// string because the cephv1 phase constants are a mix of ConditionType and +// ConditionReason (e.g. string(cephv1.ReconcileFailed)). +func ObjectStoreUserPhase(phase string) func(*cephv1.CephObjectStoreUser) bool { + return func(u *cephv1.CephObjectStoreUser) bool { + return u.Status != nil && u.Status.Phase == phase + } +} + +// BucketTopic reports whether a CephBucketTopic has been reconciled to Ready +// with its ARN populated. Note that .status.ARN is nil while .status.phase == +// Reconciling, so the phase check alone is not sufficient. +func BucketTopic(bt *cephv1.CephBucketTopic) bool { + return BucketTopicPhase(string(cephv1.ConditionReady))(bt) && bt.Status.ARN != nil +} + +// BucketTopicPhase returns a predicate reporting whether a CephBucketTopic's +// .status.phase matches phase. The parameter is a plain string because the +// cephv1 phase constants are a mix of ConditionType and ConditionReason (e.g. +// string(cephv1.ReconcileFailed)). +func BucketTopicPhase(phase string) func(*cephv1.CephBucketTopic) bool { + return func(bt *cephv1.CephBucketTopic) bool { + return bt.Status != nil && bt.Status.Phase == phase + } +} + +// OBCBound reports whether an ObjectBucketClaim has reached the Bound phase. +func OBCBound(obc *bktv1alpha1.ObjectBucketClaim) bool { + return obc.Status.Phase == bktv1alpha1.ObjectBucketClaimStatusPhaseBound +} + +// OBBound reports whether an ObjectBucket has reached the Bound phase. +func OBBound(ob *bktv1alpha1.ObjectBucket) bool { + return ob.Status.Phase == bktv1alpha1.ObjectBucketStatusPhaseBound +} diff --git a/tests/integration/object/util/wait4/wait4.go b/tests/integration/object/util/wait4/wait4.go new file mode 100644 index 000000000..f4a6d7e8e --- /dev/null +++ b/tests/integration/object/util/wait4/wait4.go @@ -0,0 +1,488 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wait4 + +import ( + "bufio" + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/cache" + watchtools "k8s.io/client-go/tools/watch" + + "github.com/rook/rook/tests/framework/utils" +) + +// Default timeouts for waiting on operator reconciliation in the object +// tests. Bespoke waits (e.g. the shared store's multi-minute teardown ladder) +// still pass explicit durations. +const ( + // TimeoutShort covers routine status changes on an already-running + // operator and rgw: condition waits, deletions, field updates. + TimeoutShort = 40 * time.Second + // TimeoutMedium covers resource creation that needs a full reconcile, + // such as a CephBucketTopic becoming Ready with its ARN set. + TimeoutMedium = time.Minute + // TimeoutLong covers first-reconcile waits that may race rgw startup, + // such as CephObjectStoreUser creation right after the store comes up. + TimeoutLong = 2 * time.Minute +) + +// NamespacedWatcher is satisfied by any typed namespaced client (corev1 +// ServiceInterface, cephv1 CephObjectStoreUserInterface, etc.). T is the +// resource's pointer type and L its list type, both inferred at the call site +// from the client's Get/List return types. +type NamespacedWatcher[T, L runtime.Object] interface { + Get(ctx context.Context, name string, opts metav1.GetOptions) (T, error) + List(ctx context.Context, opts metav1.ListOptions) (L, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) +} + +// nameScopedListWatch returns a ListWatch restricted to a single resource name. +func nameScopedListWatch[T, L runtime.Object](ctx context.Context, client NamespacedWatcher[T, L], name string) *cache.ListWatch { + fs := fields.OneTermEqualSelector("metadata.name", name).String() + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fs + return client.List(ctx, opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fs + return client.Watch(ctx, opts) + }, + } +} + +// heartbeatInterval is how often a watch-based wait logs progress while it is +// still blocked. +const heartbeatInterval = 15 * time.Second + +// heartbeat logs a periodic progress line for a watch-based wait that would +// otherwise block silently, so a slow reconcile shows up in CI output instead +// of looking like a hang. It logs nothing for waits shorter than +// heartbeatInterval; the caller must invoke the returned stop func when the +// wait returns. +func heartbeat(t *testing.T, desc string) (stop func()) { + t.Helper() + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(heartbeatInterval) + defer ticker.Stop() + for elapsed := heartbeatInterval; ; elapsed += heartbeatInterval { + select { + case <-done: + return + case <-ticker.C: + t.Logf("still waiting for %s (%s)", desc, elapsed) + } + } + }() + return func() { close(done) } +} + +// waitForAbsent blocks until the named resource is gone or the timeout elapses. +// +// The wait is race-free against fast deletions: UntilWithSync performs an +// initial List (scoped to the resource name via a field selector) before +// starting the watch, and the precondition treats an empty store as "already +// gone". Subsequent deletion events are guaranteed to be delivered because the +// watch resumes from the List's resourceVersion. +func waitForAbsent[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedWatcher[T, L], name string, timeout time.Duration) error { + var objType T + lw := nameScopedListWatch(ctx, client, name) + + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + precondition := func(store cache.Store) (bool, error) { + // lw is name-scoped, so an empty store unambiguously means the object + // was already gone by the time the reflector's initial list completed. + return len(store.List()) == 0, nil + } + condition := func(ev watch.Event) (bool, error) { + return ev.Type == watch.Deleted, nil + } + + defer heartbeat(t, fmt.Sprintf("%T %q to be removed", objType, name))() + + if _, err := watchtools.UntilWithSync(waitCtx, lw, objType, precondition, condition); err != nil { + return fmt.Errorf("%T %q is still present: %w", objType, name, err) + } + return nil +} + +// waitForCondition blocks until the named resource satisfies cond or the timeout +// elapses, returning the matching object. The initial List makes it race-free in +// the same way as waitForAbsent: a resource that already satisfies cond is +// detected immediately, and subsequent updates arrive as watch events. +func waitForCondition[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedWatcher[T, L], name string, cond func(T) bool, timeout time.Duration) (T, error) { + var objType T + var match T + lw := nameScopedListWatch(ctx, client, name) + + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + precondition := func(store cache.Store) (bool, error) { + for _, o := range store.List() { + if typed, ok := o.(T); ok && cond(typed) { + match = typed + return true, nil + } + } + return false, nil + } + condition := func(ev watch.Event) (bool, error) { + switch ev.Type { + case watch.Added, watch.Modified: + if typed, ok := ev.Object.(T); ok && cond(typed) { + match = typed + return true, nil + } + case watch.Deleted: + return false, fmt.Errorf("%T %q was deleted while waiting for condition", objType, name) + } + return false, nil + } + + defer heartbeat(t, fmt.Sprintf("%T %q to be ready", objType, name))() + + if _, err := watchtools.UntilWithSync(waitCtx, lw, objType, precondition, condition); err != nil { + return match, fmt.Errorf("%T %q did not satisfy condition: %w", objType, name, err) + } + return match, nil +} + +// NamespacedDeleter is a NamespacedWatcher that can also delete by name. +type NamespacedDeleter[T, L runtime.Object] interface { + NamespacedWatcher[T, L] + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error +} + +func deleteAndWait[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedDeleter[T, L], name string, timeout time.Duration) error { + var objType T + if err := client.Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete %T %q: %w", objType, name, err) + } + return waitForAbsent[T, L](ctx, t, client, name, timeout) +} + +// AssertDelete deletes a single named resource and blocks until it is gone, +// reporting failures (including timeout) via assert.Fail (non-fatal). It returns +// true on success. Prefer this for teardown/cleanup, so a stuck finalizer on one +// resource does not prevent cleanup of the rest; use RequireDelete when later +// steps depend on the deletion having completed. +func AssertDelete[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedDeleter[T, L], + name string, + timeout time.Duration, + msgAndArgs ...interface{}, +) bool { + t.Helper() + if err := deleteAndWait(ctx, t, client, name, timeout); err != nil { + return assert.Fail(t, err.Error(), msgAndArgs...) + } + return true +} + +// RequireDelete is the fatal counterpart of AssertDelete: it stops the test +// (FailNow) if the resource cannot be deleted or is not gone before the timeout. +func RequireDelete[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedDeleter[T, L], + name string, + timeout time.Duration, + msgAndArgs ...interface{}, +) { + t.Helper() + require.NoError(t, deleteAndWait(ctx, t, client, name, timeout), msgAndArgs...) +} + +// AssertAbsent waits for a named resource to disappear without deleting it +// itself, reporting failures via assert.Fail (non-fatal). Use it for resources +// removed as a side effect of another action — e.g. an ObjectBucket that is +// garbage-collected when its ObjectBucketClaim is deleted. To delete and wait, +// use AssertDelete/RequireDelete. +func AssertAbsent[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedWatcher[T, L], + name string, + timeout time.Duration, + msgAndArgs ...interface{}, +) bool { + t.Helper() + if err := waitForAbsent(ctx, t, client, name, timeout); err != nil { + return assert.Fail(t, err.Error(), msgAndArgs...) + } + return true +} + +// RequireAbsent is the fatal counterpart of AssertAbsent. +func RequireAbsent[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedWatcher[T, L], + name string, + timeout time.Duration, + msgAndArgs ...interface{}, +) { + t.Helper() + require.NoError(t, waitForAbsent(ctx, t, client, name, timeout), msgAndArgs...) +} + +// NamespacedReadyWaiter is a NamespacedWatcher that can also create. +type NamespacedReadyWaiter[T, L runtime.Object] interface { + NamespacedWatcher[T, L] + Create(ctx context.Context, obj T, opts metav1.CreateOptions) (T, error) +} + +// createAndWaitReady creates obj and, when ready is non-nil, blocks until ready +// reports true for the live object or the timeout elapses, returning the live +// object. Pass a nil ready predicate for resources that are usable as soon as +// they are created (Namespaces, Services, Secrets, ...). +// +// Unlike deletion, "ready" has no universal representation across resource +// types, so the readiness check is supplied by the caller. +func createAndWaitReady[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedReadyWaiter[T, L], + obj T, + ready func(T) bool, + timeout time.Duration, +) (T, error) { + var zero T + + created, err := client.Create(ctx, obj, metav1.CreateOptions{}) + if err != nil { + return zero, fmt.Errorf("failed to create %T: %w", obj, err) + } + if ready == nil { + return created, nil + } + + accessor, err := meta.Accessor(created) + if err != nil { + return zero, fmt.Errorf("failed to read metadata of %T: %w", created, err) + } + + return waitForCondition[T, L](ctx, t, client, accessor.GetName(), ready, timeout) +} + +// AssertCreate creates obj and waits for the ready predicate, reporting failures +// (including timeout) via assert.Fail (non-fatal) like AssertDelete. It returns +// the live object and true on success. Prefer this for cleanup/best-effort +// creation where the test should continue on failure; for setup preconditions +// use RequireCreate. +func AssertCreate[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedReadyWaiter[T, L], + obj T, + ready func(T) bool, + timeout time.Duration, + msgAndArgs ...interface{}, +) (T, bool) { + t.Helper() + live, err := createAndWaitReady(ctx, t, client, obj, ready, timeout) + if err != nil { + var zero T + assert.Fail(t, err.Error(), msgAndArgs...) + return zero, false + } + return live, true +} + +// RequireCreate is the fatal counterpart of AssertCreate: it stops the test +// (FailNow) if the resource cannot be created or does not become ready, and +// returns the live object. This is the common case for setup preconditions, so +// the call site needs no follow-up require.True. +func RequireCreate[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedReadyWaiter[T, L], + obj T, + ready func(T) bool, + timeout time.Duration, + msgAndArgs ...interface{}, +) T { + t.Helper() + live, err := createAndWaitReady(ctx, t, client, obj, ready, timeout) + require.NoError(t, err, msgAndArgs...) + return live +} + +// AssertCondition waits for the named resource to satisfy cond, reporting +// failures (including timeout) via assert.Fail (non-fatal). It returns the +// matching object and true on success. Use it to wait on an already-created +// resource — e.g. an OBC reaching Bound, or a status field changing after an +// update; for the create-and-wait case use AssertCreate/RequireCreate. +func AssertCondition[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedWatcher[T, L], + name string, + cond func(T) bool, + timeout time.Duration, + msgAndArgs ...interface{}, +) (T, bool) { + t.Helper() + match, err := waitForCondition(ctx, t, client, name, cond, timeout) + if err != nil { + var zero T + assert.Fail(t, err.Error(), msgAndArgs...) + return zero, false + } + return match, true +} + +// RequireCondition is the fatal counterpart of AssertCondition. +func RequireCondition[T, L runtime.Object]( + ctx context.Context, + t *testing.T, + client NamespacedWatcher[T, L], + name string, + cond func(T) bool, + timeout time.Duration, + msgAndArgs ...interface{}, +) T { + t.Helper() + match, err := waitForCondition(ctx, t, client, name, cond, timeout) + require.NoError(t, err, msgAndArgs...) + return match +} + +// AssertEventually polls cond until it returns nil or the timeout elapses, +// reporting failure via assert.Fail (non-fatal). It is for state the +// Kubernetes API cannot watch (the rgw admin API, S3, SNS); k8s resources +// should use the watch-based helpers above. The poll loop is utils.Eventually; +// this adds the assert-flavored, non-fatal reporting that pairs with the +// AssertCreate/AssertDelete/AssertCondition family. +// +// cond must not call assert or require: report transient failures by returning +// an error (which is surfaced in the timeout message), and make assertions on +// captured state after the wait. cond receives a context bounded by the wait +// deadline; thread it into the reads it performs. +func AssertEventually(ctx context.Context, t *testing.T, timeout time.Duration, desc string, cond func(context.Context) error) bool { + t.Helper() + if err := utils.Eventually(ctx, t, timeout, desc, cond); err != nil { + return assert.Fail(t, err.Error()) + } + return true +} + +// RequireEventually is the fatal counterpart of AssertEventually: it stops +// the test (FailNow) if cond does not succeed before the timeout. Use it +// when subsequent statements consume the polled result. +func RequireEventually(ctx context.Context, t *testing.T, timeout time.Duration, desc string, cond func(context.Context) error) { + t.Helper() + require.NoError(t, utils.Eventually(ctx, t, timeout, desc, cond)) +} + +// matchPodLog follows the logs of the first pod matching selector until match +// returns true for a line, the stream ends, or the wait deadline passes. +// Reaching the end of the stream without a match is an error: following the +// stream (rather than scanning a snapshot) is what lets the wait distinguish +// "absent" from "not logged yet". +func matchPodLog(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, selector labels.Selector, timeout time.Duration, match func(line string) bool) error { + t.Helper() + + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(waitCtx, metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return fmt.Errorf("failed to list pods: %w", err) + } + if len(pods.Items) == 0 { + return fmt.Errorf("no pods found with labels %v in namespace %q", selector, namespace) + } + + // if there are multiple pods, just pick the first one + pod := pods.Items[0] + t.Logf("matching against logs of pod %q (first match for labels %v)", pod.Name, selector) + + req := k8sh.Clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Follow: true}) + logStream, err := req.Stream(waitCtx) + if err != nil { + return fmt.Errorf("failed to stream logs from pod %q: %w", pod.Name, err) + } + defer logStream.Close() + + scanner := bufio.NewScanner(logStream) + for scanner.Scan() { + if match(scanner.Text()) { + return nil + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("no matching log line from pod %q: %w", pod.Name, err) + } + return fmt.Errorf("log stream from pod %q ended without a matching line", pod.Name) +} + +// AssertPodLog follows the logs of the first pod matching selector and reports +// failure via assert.Fail (non-fatal) if no line satisfies match before the +// timeout. It is for log-based state the Kubernetes API cannot watch (e.g. an +// operator logging a reconcile error); for a setup precondition use +// RequirePodLog. +func AssertPodLog( + ctx context.Context, + t *testing.T, + k8sh *utils.K8sHelper, + namespace string, + selector labels.Selector, + timeout time.Duration, + match func(line string) bool, + msgAndArgs ...interface{}, +) bool { + t.Helper() + if err := matchPodLog(ctx, t, k8sh, namespace, selector, timeout, match); err != nil { + return assert.Fail(t, err.Error(), msgAndArgs...) + } + return true +} + +// RequirePodLog is the fatal counterpart of AssertPodLog. +func RequirePodLog( + ctx context.Context, + t *testing.T, + k8sh *utils.K8sHelper, + namespace string, + selector labels.Selector, + timeout time.Duration, + match func(line string) bool, + msgAndArgs ...interface{}, +) { + t.Helper() + require.NoError(t, matchPodLog(ctx, t, k8sh, namespace, selector, timeout, match), msgAndArgs...) +}