Merge pull request #17852 from rook/mergify/bp/release-1.20/pr-17663

test: improve the shared-store object suite tests (backport #17663)

tests/integration/ceph_object_test.go

@@ -209,20 +209,15 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
 	// now test operation of the first object store
 	testObjectStoreOperations(s, helper, k8sh, settings, storeName, swiftAndKeystone)
 
-	// The namespaced test packages below all skip when TLS is enabled, so only set up the
-	// shared store when it will actually be used.
-	var sharedObjectStore *cephv1.CephObjectStore
-	if !tlsEnable {
-		var teardownSharedStore func()
-		sharedObjectStore, teardownSharedStore = sharedstore.Setup(s.T(), k8sh)
-		defer teardownSharedStore()
+	sharedObjectStore := sharedstore.Create(s.T(), k8sh, installer, tlsEnable,
+		bucketowner.Namespace, userkeys.Namespace, topickafka.Namespace, useropmask.Namespace, usercaps.Namespace)
+	defer sharedObjectStore.Destroy()
 
-		bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore)
-		userkeys.TestObjectStoreUserKeys(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore)
-		topickafka.TestBucketTopicKafka(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore)
-		useropmask.TestObjectStoreUserOpMask(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore)
-		usercaps.TestObjectStoreUserCaps(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore)
-	}
+	bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, sharedObjectStore)
+	userkeys.TestObjectStoreUserKeys(s.T(), k8sh, sharedObjectStore)
+	topickafka.TestBucketTopicKafka(s.T(), k8sh, sharedObjectStore)
+	useropmask.TestObjectStoreUserOpMask(s.T(), k8sh, sharedObjectStore)
+	usercaps.TestObjectStoreUserCaps(s.T(), k8sh, sharedObjectStore)
 
 	bucketNotificationTestStoreName := "bucket-notification-" + storeName
 	createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable, swiftAndKeystone)

tests/integration/object/README.md

@@ -0,0 +1,215 @@
+# Object storage integration tests
+
+This tree holds the new-style integration tests for rook's object storage
+features. It is the model for converting the remaining old-style object tests
+(`testObjectStoreOperations`, store lifecycle, bucket notifications, COSI);
+follow the conventions here exactly so the conversion happens once.
+
+## Execution model
+
+- One Kubernetes cluster is shared by the whole `TestCephObjectSuite` run. The
+  testify suite in `tests/integration/ceph_object_test.go` owns cluster
+  lifecycle and dispatches into the packages here; these packages are plain Go
+  libraries, **not** standalone `go test` targets.
+- A single shared `CephObjectStore` fixture (`util/sharedstore`) is created
+  once per suite pass and torn down after all packages run. It provides the
+  store CR, a go-ceph rgw admin client, and an SNS client, and is created with
+  or without TLS to match the pass. The packages run in both passes: the
+  with/without-TLS split runs them as separate parallel jobs, so the TLS pass
+  adds no serial runtime to a single job.
+- Packages run **sequentially** and each test is an ordered script of
+  subtests. Do not use `t.Parallel`: the steps depend on prior steps, and the
+  cluster is shared.
+
+## Layout
+
+The tree mirrors `pkg/operator/ceph/object/*`: one top-level directory per
+operator package, with a feature subdirectory layer when multiple test
+packages cover subsets of one operator package. The operator ROOT package
+(the CephObjectStore controller itself) maps to the tree root,
+`tests/integration/object/` itself, not a subdirectory. Leaf package names
+must not collide with std-lib package names (no `io`, no `http`).
+
+| test package | operator package | covers |
+|---|---|---|
+| `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling |
+| `topic/kafka` | `object/topic` | CephBucketTopic kafka endpoints |
+| `user/caps` | `object/user` | user capabilities |
+| `user/keys` | `object/user` | explicit S3 key management |
+| `user/opmask` | `object/user` | user op_mask |
+| reserved: `bucket/rw`, `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents`, `notification/*`, `cosi` | | future conversions |
+
+Shared utilities live under `util/`:
+
+- `wait4` — waits for object-test state: watch-based `Assert/Require` ×
+  `Create/Delete/Condition/Absent` for k8s resources, `Assert/RequireEventually`
+  polling for non-k8s state, `Assert/RequirePodLog` for a matching line in a
+  pod's log stream, and the shared timeout tiers.
+- `ready` — readiness predicates for the wait4 helpers (`ObjectStoreUser`,
+  `BucketTopic`, `OBCBound`, ...).
+- `fixture` — create-with-`t.Cleanup` helpers for pure-cleanup resources
+  (namespaces, StorageClasses).
+- `obc` — pure constructors for the resources ObjectBucketClaim tests build
+  (the provisioner `StorageClass`; OBCs and per-OBC S3 clients as `bucket/*`
+  is converted).
+- `secrets` — verification helpers for the Secret references object CRDs
+  publish in their status, shared by more than one package.
+- `sharedstore` — the shared CephObjectStore fixture.
+- `admin`, `sns`, `s3`, `tls` — client builders and TLS cert generation.
+
+## Anatomy of a package
+
+One exported entry function per package, called from the dispatcher:
+
+```go
+func TestObjectStoreUserCaps(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
+	var (
+		defaultName = "test-usercaps"
+		objectStore = store.ObjectStore()
+		adminClient = store.AdminClient()
+
+		ns = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: defaultName}}
+
+		osu1 = cephv1.CephObjectStoreUser{ /* fixture literal or constructor */ }
+
+		osuClient = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name)
+	)
+
+	t.Run("ObjectStoreUser caps", func(t *testing.T) {
+		ctx := t.Context()
+
+		fixture.RequireNamespace(t, k8sh, ns)
+
+		t.Run(fmt.Sprintf("create CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
+			wait4.RequireCreate(ctx, t, osuClient, &osu1, ready.ObjectStoreUser, wait4.TimeoutLong)
+		})
+
+		// ... ordered subtests: act, verify, act, verify ...
+	})
+}
+```
+
+Rules encoded in that shape:
+
+- The entry signature is `(t, k8sh, store)` and never changes. New
+  dependencies are added as `Sharedstore` accessors, not parameters.
+- The var block declares everything the test uses: fixture objects (use
+  constructors — `obc.StorageClass`, package-local helpers like
+  `awsKeySecret`/`objectUserKey` — to keep literals one line each), then the
+  typed clients bound to locals (`osuClient`, `obcClient`, `obClient`, ...).
+  Spelled-out client chains do not appear in test bodies.
+- Exactly one outer `t.Run` per package, named `"<Subject> <feature>"`. It
+  must be unique across all object packages: it is the `-run` filter handle.
+- `ctx := t.Context()` is the first line inside the outer `t.Run` — contexts
+  are test-scoped, so nothing outlives its test. (Helpers that wrap their own
+  `t.Run` bind their own `ctx := t.Context()` from the subtest's `t`.)
+- `fixture.Require*` calls come next. Fixture teardown runs via `t.Cleanup`,
+  so it happens even when the test fails partway. Only PURE-CLEANUP teardown
+  belongs there; a deletion that asserts behavior ("deleting the OBC must not
+  delete the user") is an ordered subtest.
+- Repeated multi-assert verification goes in a package-local
+  `check<Aspect>(t, deps, subject, expected ...)` helper that wraps its own
+  `t.Run` and takes the expected state variadically. Promote a helper to a
+  shared `util/` package named for its subject (e.g. `util/secrets`) only
+  when a second package needs it.
+
+## Waiting
+
+- State visible to the Kubernetes API uses the watch-based `wait4` verbs:
+  `RequireCreate` (create + wait ready), `AssertDelete`/`RequireDelete`
+  (delete + wait gone), `RequireCondition`/`AssertCondition` (wait for a
+  predicate on an existing resource), `AssertAbsent`/`RequireAbsent` (wait
+  for cascade deletion you did not issue). Predicates come from `util/ready`
+  or are inlined when test-specific.
+- State NOT visible to the Kubernetes API (rgw admin ops, S3, SNS) uses
+  `wait4.AssertEventually`/`RequireEventually`, the assert/require-flavored
+  wrappers over the framework's `utils.Eventually` poll loop. Poll closures
+  must not call `assert` or `require` — return an error on transient failures
+  (it is surfaced in the timeout message) and assert on captured state after
+  the wait (capture-on-success: assign an outer variable inside the closure
+  just before returning nil). The closure receives a context bounded by the
+  wait deadline; thread it into the reads it performs.
+- Log-based state (an operator or endpoint logging a line) uses
+  `wait4.AssertPodLog`/`RequirePodLog`, which follows the first matching pod's
+  log stream until a line satisfies the predicate or the timeout elapses.
+- Timeouts come from the shared tiers: `wait4.TimeoutShort` (routine status
+  changes), `TimeoutMedium` (full reconcile, e.g. topic Ready with ARN),
+  `TimeoutLong` (first reconcile that may race rgw startup). Bespoke waits
+  (e.g. the shared store teardown ladder) pass explicit durations.
+
+## assert vs require
+
+- `require` (and `wait4.Require*`) for anything the rest of the (sub)test
+  cannot proceed without: creates, fetches of the object about to be
+  inspected, decoding, setup waits. Fail fast to avoid cascade noise.
+- `assert` (and `wait4.Assert*`) for the properties under test — so every
+  violated property in a subtest is reported together — and for teardown
+  deletes, so one stuck finalizer does not strand the rest of cleanup.
+- `wait4.RequireDelete` only when later steps depend on the deletion having
+  completed.
+- Never `assert`/`require` inside an `Eventually` poll closure.
+
+## Naming
+
+- Subtest names are lowercase sentences: imperative for actions
+  (`"create CephObjectStoreUser %q"`, `"delete obc %q"`), declarative for
+  assertions (`"obc %q has bucketOwner %q set"`, `"no secrets in ns %q"`).
+  Resource names are always interpolated with `%q`. The `-v` output of a run
+  should read as the full scenario.
+
+## Running and filtering
+
+CI runs the suite as:
+
+```
+go test -v -timeout 2400s -failfast -run CephObjectSuite github.com/rook/rook/tests/integration
+```
+
+To select one package's tests through the suite:
+
+```
+go test -tags ceph_preview -run 'TestCephObjectSuite/TestWithoutTLS/ObjectStoreUser_keys' ./tests/integration
+```
+
+All build/vet/lint commands need the `ceph_preview` build tag or the module
+does not type-check:
+
+```
+go build -tags ceph_preview ./tests/...
+go vet -tags ceph_preview ./tests/integration/...
+gofmt -l tests/integration
+```
+
+These are cluster-backed integration tests; the only end-to-end validation is
+the CI object suite.
+
+## Adding a new package (checklist)
+
+1. Create `tests/integration/object/<operator-pkg>/<feature>/` per the layout
+   rules above.
+2. Write the entry func following the anatomy section; pick a globally unique
+   outer `t.Run` name.
+3. Wire one dispatcher line in `tests/integration/ceph_object_test.go` and,
+   if the package creates CephObjectStoreUsers, add its namespace to the
+   shared store's `AllowUsersInNamespaces`.
+4. Verify with the build/vet/gofmt commands above.
+
+## Conversion playbook (old-style tests)
+
+Per old test: enumerate its behaviors; pick or create target package(s);
+express fixtures as var-block constructors; map k8s waits to `wait4` verbs
+and non-k8s polls to `Eventually`; write check* helpers for repeated
+verification; wire the dispatcher; delete the old code; retire
+`tests/framework/clients` methods whose last consumer is gone.
+
+| old test | target package(s) | still needs (build in that PR) |
+|---|---|---|
+| `testObjectStoreOperations` S3 I/O + OBC-stays-Bound | `bucket/rw` | per-OBC S3 client (TLS-aware, from OBC secret) |
+| `testObjectStoreOperations` user/bucket quotas | `bucket/quota` | per-OBC S3 client |
+| `testObjectStoreOperations` bucket policy | `bucket/policy` | per-OBC S3 client |
+| `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | per-OBC S3 client, move `lifecycleCmpOpts` out of the dispatcher |
+| `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `ready` |
+| `createCephObjectStore`/`runObjectE2ETestLite`/deletion asserts + zone.json canary | `tests/integration/object/lifecycle` | store create/health/delete helpers, `Sharedstore.Installer()` accessor |
+| `testBucketNotifications` | `notification/*` | HTTP endpoint fixture, per-OBC S3 client (the pod-log waiter is now `wait4.Assert/RequirePodLog`) |
+| `testCOSIDriver` | `cosi` | `kubectl -k` install fixture, COSI readiness waits |
+| upgrade-suite object usage | stays in upgrade suite | switch to typed clients + `wait4`/`ready` |

tests/integration/object/bucket/owner/owner.go

@@ -17,79 +17,34 @@ limitations under the License.
 package owner
 
 import (
-	"bufio"
 	"context"
 	"fmt"
-	"log"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/ceph/go-ceph/rgw/admin"
-	"github.com/coreos/pkg/capnslog"
 	bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
-	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/tests/framework/installer"
-	"github.com/rook/rook/tests/framework/utils"
-	utiladmin "github.com/rook/rook/tests/integration/object/util/admin"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
-	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/tests/framework/utils"
+	"github.com/rook/rook/tests/integration/object/util/fixture"
+	"github.com/rook/rook/tests/integration/object/util/sharedstore"
+	"github.com/rook/rook/tests/integration/object/util/wait4"
 )
 
-func WaitForPodLogContainingText(k8sh *utils.K8sHelper, namespace string, selector *labels.Selector, text string, timeout time.Duration) error {
-	pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(
-		context.Background(),
-		metav1.ListOptions{
-			LabelSelector: (*selector).String(),
-		},
-	)
-	if err != nil {
-		return fmt.Errorf("failed to list pods: %v", err)
-	}
+const Namespace = "test-bucketowner"
 
-	if len(pods.Items) == 0 {
-		return fmt.Errorf("no pods found with labels %v in namespace %q", selector, namespace)
-	}
-
-	// if there are multiple pods, just pick the first one
-	selectedPod := pods.Items[0]
-	log.Printf("Found pod %q (first match) with labels %v\n", selectedPod.Name, *selector)
-
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-
-	req := k8sh.Clientset.CoreV1().Pods(namespace).GetLogs(selectedPod.Name, &corev1.PodLogOptions{})
-
-	logStream, err := req.Stream(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to stream logs from pod %q: %v", selectedPod.Name, err)
-	}
-	defer logStream.Close()
-
-	scanner := bufio.NewScanner(logStream)
-
-	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.Contains(line, text) {
-			break
-		}
-	}
-	// Check for scanner error (could be context timeout, etc.)
-	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("error reading log stream: %v", err)
-	}
-
-	return nil
-}
-
-func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) {
+func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
 	var (
-		defaultName = "test-bucketowner"
+		defaultName = Namespace
+		objectStore = store.ObjectStore()
+		adminClient = store.AdminClient()
 
 		ns = &corev1.Namespace{
 			ObjectMeta: metav1.ObjectMeta{
@@ -97,16 +52,7 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 			},
 		}
 
-		storageClass = &storagev1.StorageClass{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: defaultName,
-			},
-			Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
-			Parameters: map[string]string{
-				"objectStoreName":      objectStore.Name,
-				"objectStoreNamespace": objectStore.Namespace,
-			},
-		}
+		storageClass = fixture.StorageClass(defaultName, objectStore)
 
 		// test user without any quotas set
 		osu1 = cephv1.CephObjectStoreUser{
@@ -178,102 +124,48 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 				},
 			},
 		}
+
+		osuClient = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name)
+		obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name)
+		obClient  = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
 	)
 	t.Run("OBC bucketOwner", func(t *testing.T) {
-		if tlsEnable {
-			// There is lots of coverage of rgw working with tls enabled; skipping to save time.
-			// If tls is to be enabled, cert generation needs to be added and a
-			// different CephObjectStore name needs to be set for with/without tls as
-			// CephObjectStore does not currently cleanup the rgw realm.
-			t.Skip("skipping test for TLS enabled clusters")
-		}
+		ctx := t.Context()
 
-		var adminClient *admin.API
-		ctx := context.TODO()
-
-		t.Run(fmt.Sprintf("create ns %q", ns.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create sc %q", storageClass.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.StorageV1().StorageClasses().Create(ctx, storageClass, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
+		fixture.RequireNamespace(t, k8sh, ns)
+		fixture.RequireStorageClass(t, k8sh, storageClass)
 
 		// since this is an obc specific subtest we assume that CephObjectStoreUser
 		// is working and the rgw service state does not need to be inspected to
 		// confirm user creation.
 		t.Run(fmt.Sprintf("create CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Create(ctx, &osu1, metav1.CreateOptions{})
-			require.NoError(t, err)
-
 			// user creation may be slow right after rgw start up
-			osuReady := utils.Retry(120, time.Second, "CephObjectStoreUser is Ready", func() bool {
-				liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				if liveOsu.Status == nil {
-					return false
-				}
-
-				return liveOsu.Status.Phase == "Ready"
-			})
-			require.True(t, osuReady)
+			wait4.RequireCreate(ctx, t, osuClient, &osu1, wait4.ObjectStoreUser, wait4.TimeoutLong)
 		})
 
 		t.Run(fmt.Sprintf("create obc %q with bucketOwner %q", obc1.Name, obc1.Spec.AdditionalConfig["bucketOwner"]), func(t *testing.T) {
-			_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Create(ctx, &obc1, metav1.CreateOptions{})
+			_, err := obcClient.Create(ctx, &obc1, metav1.CreateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("obc %q has bucketOwner %q set", obc1.Name, obc1.Spec.AdditionalConfig["bucketOwner"]), func(t *testing.T) {
-			var liveObc *bktv1alpha1.ObjectBucketClaim
-			obcBound := utils.Retry(40, time.Second, "OBC is Bound", func() bool {
-				var err error
-				liveObc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				return liveObc.Status.Phase == bktv1alpha1.ObjectBucketClaimStatusPhaseBound
-			})
-			require.True(t, obcBound)
+			liveObc := wait4.RequireCondition(ctx, t, obcClient, obc1.Name, wait4.OBCBound, wait4.TimeoutShort)
 
 			// verify that bucketOwner is set on the live obc
 			assert.Equal(t, obc1.Spec.AdditionalConfig["bucketOwner"], liveObc.Spec.AdditionalConfig["bucketOwner"])
 		})
 
 		t.Run(fmt.Sprintf("ob for obc %q has bucketOwner %q set", obc1.Name, obc1.Spec.AdditionalConfig["bucketOwner"]), func(t *testing.T) {
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			var liveOb *bktv1alpha1.ObjectBucket
-			obBound := utils.Retry(40, time.Second, "OB is Bound", func() bool {
-				var err error
-				liveOb, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				return liveOb.Status.Phase == bktv1alpha1.ObjectBucketStatusPhaseBound
-			})
-			require.True(t, obBound)
+			liveOb := wait4.RequireCondition(ctx, t, obClient, obName, wait4.OBBound, wait4.TimeoutShort)
 
 			// verify that bucketOwner is set on the live ob
 			assert.Equal(t, obc1.Spec.AdditionalConfig["bucketOwner"], liveOb.Spec.Connection.AdditionalState["bucketOwner"])
 		})
 
-		// the rgw admin api is used to verify bucket ownership
-		t.Run("setup rgw admin api client", func(t *testing.T) {
-			var err error
-			adminClient, err = utiladmin.NewAdminClient(objectStore, installer, k8sh, tlsEnable)
-			require.NoError(t, err)
-		})
-
 		t.Run(fmt.Sprintf("bucket created with owner %q", obc1.Spec.AdditionalConfig["bucketOwner"]), func(t *testing.T) {
 			bucket, err := adminClient.GetBucketInfo(ctx, admin.Bucket{Bucket: obc1.Spec.BucketName})
 			require.NoError(t, err)
@@ -294,38 +186,23 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 
 		t.Run(fmt.Sprintf("create CephObjectStoreUser %q", osu2.Name), func(t *testing.T) {
 			// create user2
-			_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Create(ctx, &osu2, metav1.CreateOptions{})
-			require.NoError(t, err)
-
-			osuReady := utils.Retry(40, time.Second, "CephObjectStoreUser is Ready", func() bool {
-				liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu2.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				if liveOsu.Status == nil {
-					return false
-				}
-
-				return liveOsu.Status.Phase == "Ready"
-			})
-			require.True(t, osuReady)
+			wait4.RequireCreate(ctx, t, osuClient, &osu2, wait4.ObjectStoreUser, wait4.TimeoutLong)
 		})
 
 		t.Run(fmt.Sprintf("update obc %q to bucketOwner %q", obc1.Name, osu2.Name), func(t *testing.T) {
 			// update obc bucketOwner
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveObc.Spec.AdditionalConfig["bucketOwner"] = osu2.Name
 
-			_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Update(ctx, liveObc, metav1.UpdateOptions{})
+			_, err = obcClient.Update(ctx, liveObc, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("obc %q has bucketOwner %q set", obc1.Name, osu2.Name), func(t *testing.T) {
 			// obc .Status.Phase does not appear to change when updating the obc
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			// verify that bucketOwner is set on the live obc
@@ -334,39 +211,32 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 
 		t.Run(fmt.Sprintf("ob for obc %q has bucketOwner %q set", obc1.Name, osu2.Name), func(t *testing.T) {
 			// ob .Status.Phase does not appear to change when updating the obc
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			var liveOb *bktv1alpha1.ObjectBucket
-			inSync := utils.Retry(40, time.Second, "OB is Bound", func() bool {
-				var err error
-				liveOb, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				return osu2.Name == liveOb.Spec.Connection.AdditionalState["bucketOwner"]
-			})
-			require.True(t, inSync)
+			liveOb := wait4.RequireCondition(ctx, t, obClient, obName,
+				func(ob *bktv1alpha1.ObjectBucket) bool {
+					return ob.Spec.Connection.AdditionalState["bucketOwner"] == osu2.Name
+				},
+				wait4.TimeoutShort)
 
 			// verify that bucketOwner is set on the live ob
 			assert.Equal(t, osu2.Name, liveOb.Spec.Connection.AdditionalState["bucketOwner"])
 		})
 
 		t.Run(fmt.Sprintf("bucket owner changed to %q", osu2.Name), func(t *testing.T) {
-			var bucket admin.Bucket
-			ownerSync := utils.Retry(40, time.Second, "bucket owner in sync", func() bool {
-				var err error
-				bucket, err = adminClient.GetBucketInfo(ctx, admin.Bucket{Bucket: obc1.Name})
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, fmt.Sprintf("bucket %q owner is %q", obc1.Name, osu2.Name), func(ctx context.Context) error {
+				bucket, err := adminClient.GetBucketInfo(ctx, admin.Bucket{Bucket: obc1.Name})
 				if err != nil {
-					return false
+					return err
 				}
 
-				return bucket.Owner == osu2.Name
+				if bucket.Owner != osu2.Name {
+					return fmt.Errorf("bucket owner is %q, want %q", bucket.Owner, osu2.Name)
+				}
+				return nil
 			})
-			assert.True(t, ownerSync)
-			assert.Equal(t, osu2.Name, bucket.Owner)
 		})
 
 		// obc should not modify pre-existing users
@@ -382,78 +252,68 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 
 		t.Run(fmt.Sprintf("remove obc %q bucketOwner", obc1.Name), func(t *testing.T) {
 			// update/remove obc bucketOwner
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveObc.Spec.AdditionalConfig = map[string]string{}
 
-			_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Update(ctx, liveObc, metav1.UpdateOptions{})
+			_, err = obcClient.Update(ctx, liveObc, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("obc %q has no bucketOwner", obc1.Name), func(t *testing.T) {
 			// verify that bucketOwner is unset on the live obc
-			notSet := utils.Retry(40, time.Second, "bucketOwner not set", func() bool {
-				liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				_, ok := liveObc.Spec.AdditionalConfig["bucketOwner"]
-				return !ok
-			})
-			assert.True(t, notSet)
+			wait4.AssertCondition(ctx, t, obcClient, obc1.Name,
+				func(obc *bktv1alpha1.ObjectBucketClaim) bool {
+					_, ok := obc.Spec.AdditionalConfig["bucketOwner"]
+					return !ok
+				},
+				wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("ob for obc %q has no bucketOwner", obc1.Name), func(t *testing.T) {
-			// ob .Status.Phase does not appear to change when updating the obc
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			notSet := utils.Retry(40, time.Second, "bucketOwner not set", func() bool {
-				liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				_, ok := liveOb.Spec.Connection.AdditionalState["bucketOwner"]
-				return !ok
-			})
-			assert.True(t, notSet)
+			wait4.AssertCondition(ctx, t, obClient, obName,
+				func(ob *bktv1alpha1.ObjectBucket) bool {
+					_, ok := ob.Spec.Connection.AdditionalState["bucketOwner"]
+					return !ok
+				},
+				wait4.TimeoutShort)
 		})
 
 		// the ob should retain the existing owner and not revert to a generated user
 		t.Run(fmt.Sprintf("bucket owner is still %q", osu2.Name), func(t *testing.T) {
-			var bucket admin.Bucket
-			ownerSync := utils.Retry(40, time.Second, "bucket owner in sync", func() bool {
-				var err error
-				bucket, err = adminClient.GetBucketInfo(ctx, admin.Bucket{Bucket: obc1.Name})
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, fmt.Sprintf("bucket %q owner is still %q", obc1.Name, osu2.Name), func(ctx context.Context) error {
+				bucket, err := adminClient.GetBucketInfo(ctx, admin.Bucket{Bucket: obc1.Name})
 				if err != nil {
-					return false
+					return err
 				}
 
-				return bucket.Owner == osu2.Name
+				if bucket.Owner != osu2.Name {
+					return fmt.Errorf("bucket owner is %q, want %q", bucket.Owner, osu2.Name)
+				}
+				return nil
 			})
-			assert.True(t, ownerSync)
-			assert.Equal(t, osu2.Name, bucket.Owner)
 		})
 
 		// this covers setting bucketOwner on an obc initially created without an explicit owner
 		t.Run(fmt.Sprintf("update obc %q to bucketOwner %q", obc1.Name, osu1.Name), func(t *testing.T) {
 			// update obc bucketOwner
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveObc.Spec.AdditionalConfig = map[string]string{"bucketOwner": osu1.Name}
 
-			_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Update(ctx, liveObc, metav1.UpdateOptions{})
+			_, err = obcClient.Update(ctx, liveObc, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("obc %q has bucketOwner %q set", obc1.Name, osu1.Name), func(t *testing.T) {
 			// obc .Status.Phase does not appear to change when updating the obc
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			// verify that bucketOwner is set on the live obc
@@ -462,94 +322,52 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 
 		t.Run(fmt.Sprintf("ob for obc %q has bucketOwner %q set", obc1.Name, osu1.Name), func(t *testing.T) {
 			// ob .Status.Phase does not appear to change when updating the obc
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			var liveOb *bktv1alpha1.ObjectBucket
-			inSync := utils.Retry(40, time.Second, "OB is Bound", func() bool {
-				var err error
-				liveOb, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				return osu1.Name == liveOb.Spec.Connection.AdditionalState["bucketOwner"]
-			})
-			require.True(t, inSync)
+			liveOb := wait4.RequireCondition(ctx, t, obClient, obName,
+				func(ob *bktv1alpha1.ObjectBucket) bool {
+					return ob.Spec.Connection.AdditionalState["bucketOwner"] == osu1.Name
+				},
+				wait4.TimeoutShort)
 
 			// verify that bucketOwner is set on the live ob
 			assert.Equal(t, osu1.Name, liveOb.Spec.Connection.AdditionalState["bucketOwner"])
 		})
 
 		t.Run(fmt.Sprintf("bucket owner changed to %q", osu1.Name), func(t *testing.T) {
-			var bucket admin.Bucket
-			ownerSync := utils.Retry(40, time.Second, "bucket owner in sync", func() bool {
-				var err error
-				bucket, err = adminClient.GetBucketInfo(ctx, admin.Bucket{Bucket: obc1.Name})
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, fmt.Sprintf("bucket %q owner is %q", obc1.Name, osu1.Name), func(ctx context.Context) error {
+				bucket, err := adminClient.GetBucketInfo(ctx, admin.Bucket{Bucket: obc1.Name})
 				if err != nil {
-					return false
+					return err
 				}
 
-				return bucket.Owner == osu1.Name
-			})
-			assert.True(t, ownerSync)
-			assert.Equal(t, osu1.Name, bucket.Owner)
-		})
-
-		t.Run(fmt.Sprintf("bucket owner changed to %q", osu1.Name), func(t *testing.T) {
-			var bucket admin.Bucket
-			ownerSync := utils.Retry(40, time.Second, "bucket owner in sync", func() bool {
-				var err error
-				bucket, err = adminClient.GetBucketInfo(ctx, admin.Bucket{Bucket: obc1.Name})
-				if err != nil {
-					return false
+				if bucket.Owner != osu1.Name {
+					return fmt.Errorf("bucket owner is %q, want %q", bucket.Owner, osu1.Name)
 				}
-
-				return bucket.Owner == osu1.Name
+				return nil
 			})
-			assert.True(t, ownerSync)
-			assert.Equal(t, osu1.Name, bucket.Owner)
 		})
 
 		t.Run(fmt.Sprintf("create obc %q with bucketOwner %q", obc2.Name, obc2.Spec.AdditionalConfig["bucketOwner"]), func(t *testing.T) {
-			_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Create(ctx, &obc2, metav1.CreateOptions{})
+			_, err := obcClient.Create(ctx, &obc2, metav1.CreateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("obc %q has bucketOwner %q set", obc2.Name, obc2.Spec.AdditionalConfig["bucketOwner"]), func(t *testing.T) {
-			var liveObc *bktv1alpha1.ObjectBucketClaim
-			obcBound := utils.Retry(40, time.Second, "OBC is Bound", func() bool {
-				var err error
-				liveObc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc2.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				return liveObc.Status.Phase == bktv1alpha1.ObjectBucketClaimStatusPhaseBound
-			})
-			require.True(t, obcBound)
+			liveObc := wait4.RequireCondition(ctx, t, obcClient, obc2.Name, wait4.OBCBound, wait4.TimeoutShort)
 
 			// verify that bucketOwner is set on the live obc
 			assert.Equal(t, obc2.Spec.AdditionalConfig["bucketOwner"], liveObc.Spec.AdditionalConfig["bucketOwner"])
 		})
 
 		t.Run(fmt.Sprintf("ob for obc %q has bucketOwner %q set", obc2.Name, obc2.Spec.AdditionalConfig["bucketOwner"]), func(t *testing.T) {
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc2.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc2.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			var liveOb *bktv1alpha1.ObjectBucket
-			obBound := utils.Retry(40, time.Second, "OB is Bound", func() bool {
-				var err error
-				liveOb, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				return liveOb.Status.Phase == bktv1alpha1.ObjectBucketStatusPhaseBound
-			})
-			require.True(t, obBound)
+			liveOb := wait4.RequireCondition(ctx, t, obClient, obName, wait4.OBBound, wait4.TimeoutShort)
 
 			// verify that bucketOwner is set on the live ob
 			assert.Equal(t, obc2.Spec.AdditionalConfig["bucketOwner"], liveOb.Spec.Connection.AdditionalState["bucketOwner"])
@@ -568,48 +386,26 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 
 		t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) {
 			// lookup ob name
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			// delete obc
-			err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Delete(ctx, obc1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
+			// delete obc; the backing ob is garbage-collected by the provisioner
+			wait4.AssertDelete(ctx, t, obcClient, obc1.Name, wait4.TimeoutShort)
 
-			absent := utils.Retry(40, time.Second, "OBC is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-
-			absent = utils.Retry(40, time.Second, "OB is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			wait4.AssertAbsent(ctx, t, obClient, obName, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("delete obc %q", obc2.Name), func(t *testing.T) {
 			// lookup ob name
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc2.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc2.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			// delete obc
-			err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Delete(ctx, obc2.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
+			// delete obc; the backing ob is garbage-collected by the provisioner
+			wait4.AssertDelete(ctx, t, obcClient, obc2.Name, wait4.TimeoutShort)
 
-			absent := utils.Retry(40, time.Second, "OBC is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obc2.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-
-			absent = utils.Retry(40, time.Second, "OB is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			wait4.AssertAbsent(ctx, t, obClient, obName, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("user %q was not deleted by obc %q", osu1.Name, obc1.Name), func(t *testing.T) {
@@ -629,7 +425,7 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 		// test obc creation with bucketOwner set to a non-existent user, which should fail
 		// "failure" means the obc remains in Pending state
 		t.Run(fmt.Sprintf("create obc %q with non-existent bucketOwner %q", obcBogusOwner.Name, obcBogusOwner.Spec.AdditionalConfig["bucketOwner"]), func(t *testing.T) {
-			_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Create(ctx, &obcBogusOwner, metav1.CreateOptions{})
+			_, err := obcClient.Create(ctx, &obcBogusOwner, metav1.CreateOptions{})
 			require.NoError(t, err)
 		})
 
@@ -637,14 +433,17 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 			selector := labels.SelectorFromSet(labels.Set{
 				"app": "rook-ceph-operator",
 			})
-			text := `error provisioning bucket: unable to get user \"test-bucket-owner-bogus-user\" creds: Ceph object user \"test-bucket-owner-bogus-user\" not found: NoSuchUser`
-
-			err := WaitForPodLogContainingText(k8sh, "object-ns-system", &selector, text, 10*time.Second)
-			require.NoError(t, err)
+			// match on the operator's "unable to get user ... creds" error
+			// mentioning the bogus owner rather than a hardcoded full log line,
+			// so the logger's quote encoding cannot break the match
+			bogusOwner := obcBogusOwner.Spec.AdditionalConfig["bucketOwner"]
+			wait4.RequirePodLog(ctx, t, k8sh, "object-ns-system", selector, wait4.TimeoutShort, func(line string) bool {
+				return strings.Contains(line, "unable to get user") && strings.Contains(line, bogusOwner)
+			})
 		})
 
 		t.Run(fmt.Sprintf("obc %q stays Pending", obcBogusOwner.Name), func(t *testing.T) {
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcBogusOwner.Namespace).Get(ctx, obcBogusOwner.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obcBogusOwner.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			assert.True(t, bktv1alpha1.ObjectBucketClaimStatusPhasePending == liveObc.Status.Phase)
@@ -657,57 +456,22 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
 
 		t.Run(fmt.Sprintf("delete obc %q", obcBogusOwner.Name), func(t *testing.T) {
 			// lookup ob name
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obcBogusOwner.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obcBogusOwner.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			// delete obc
-			err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Delete(ctx, obcBogusOwner.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
+			// delete obc; the backing ob is garbage-collected by the provisioner
+			wait4.AssertDelete(ctx, t, obcClient, obcBogusOwner.Name, wait4.TimeoutShort)
 
-			absent := utils.Retry(40, time.Second, "OBC is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).Get(ctx, obcBogusOwner.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-
-			absent = utils.Retry(40, time.Second, "OB is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			wait4.AssertAbsent(ctx, t, obClient, obName, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("delete CephObjectStoreUser %q", osu2.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Delete(ctx, osu2.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "CephObjectStoreUser is absent", func() bool {
-				_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu2.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			wait4.AssertDelete(ctx, t, osuClient, osu2.Name, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("delete CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Delete(ctx, osu1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "CephObjectStoreUser is absent", func() bool {
-				_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-		})
-
-		t.Run(fmt.Sprintf("delete sc %q", storageClass.Name), func(t *testing.T) {
-			err := k8sh.Clientset.StorageV1().StorageClasses().Delete(ctx, storageClass.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
+			wait4.AssertDelete(ctx, t, osuClient, osu1.Name, wait4.TimeoutShort)
 		})
 	})
 }

tests/integration/object/topic/kafka/kafka.go

@@ -22,117 +22,109 @@ import (
 	"fmt"
 	"net/url"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/sns"
-	"github.com/coreos/pkg/capnslog"
 	bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
-	storagev1 "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
-	utilsns "github.com/rook/rook/tests/integration/object/util/sns"
+	"github.com/rook/rook/tests/integration/object/util/fixture"
+	"github.com/rook/rook/tests/integration/object/util/secrets"
+	"github.com/rook/rook/tests/integration/object/util/sharedstore"
+	"github.com/rook/rook/tests/integration/object/util/wait4"
 )
 
-func checkStatusSecrets(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBucketTopic, expectedSecrets []*corev1.Secret) {
-	t.Run(fmt.Sprintf("cephBucketTopic %q has .status.secrets set", bt.Name), func(t *testing.T) {
-		ctx := context.TODO()
-
-		liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt.Namespace).Get(ctx, bt.Name, metav1.GetOptions{})
-		require.NoError(t, err)
-
-		require.NotNil(t, liveBt.Status)
-		assert.Len(t, liveBt.Status.Secrets, len(expectedSecrets))
-
-		for _, secret := range expectedSecrets {
-			secretRef, err := func(secretName string, keys []cephv1.SecretReference) (cephv1.SecretReference, error) {
-				for _, secretRef := range keys {
-					if secretRef.Name == secretName {
-						return secretRef, nil
-					}
-				}
-				return cephv1.SecretReference{}, fmt.Errorf("secretReference for secret %q not found in CephObjectStoreUser.status.keys", secret.Name)
-			}(secret.Name, liveBt.Status.Secrets)
-			require.NoError(t, err)
-
-			// fetch the live secret for UID and ResourceVersion
-			liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
-			require.NoError(t, err)
-
-			assert.Equal(t, liveSecret.Name, secretRef.Name)
-			assert.Equal(t, liveSecret.Namespace, secretRef.Namespace)
-			assert.Equal(t, liveSecret.UID, secretRef.UID)
-			assert.Equal(t, liveSecret.ResourceVersion, secretRef.ResourceVersion)
-		}
-	})
+func checkStatusSecrets(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBucketTopic, expected ...*corev1.Secret) {
+	secrets.RequireStatusRefs(t, k8sh,
+		k8sh.RookClientset.CephV1().CephBucketTopics(bt.Namespace), bt.Name,
+		fmt.Sprintf("cephBucketTopic %q has .status.secrets set", bt.Name),
+		func(topic *cephv1.CephBucketTopic) []cephv1.SecretReference {
+			if topic.Status == nil {
+				return nil
+			}
+			return topic.Status.Secrets
+		},
+		expected...)
 }
 
 func checkRgwTopicEndpoint(t *testing.T, snsClient *sns.Client, arn, user, pass string) {
 	t.Run(fmt.Sprintf("rgw topic arn %q has basic auth set", arn), func(t *testing.T) {
-		ctx := context.TODO()
+		ctx := t.Context()
 
 		var uri *url.URL
 
-		inSync := utils.Retry(40, time.Second, "rgw topic basic auth in sync", func() bool {
+		wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "rgw topic basic auth in sync", func(ctx context.Context) error {
 			topicAttrs, err := snsClient.GetTopicAttributes(ctx, &sns.GetTopicAttributesInput{
 				TopicArn: &arn,
 			})
-			require.NoError(t, err)
+			if err != nil {
+				return err
+			}
 
 			// the sns endpoint attributes are returned as JSON
 			var endpointJSON map[string]interface{}
-			err = json.Unmarshal([]byte(topicAttrs.Attributes["EndPoint"]), &endpointJSON)
-			require.NoError(t, err)
+			if err := json.Unmarshal([]byte(topicAttrs.Attributes["EndPoint"]), &endpointJSON); err != nil {
+				return err
+			}
 
-			uri, err = url.Parse(string(endpointJSON["EndpointAddress"].(string)))
-			require.NoError(t, err)
+			addr, ok := endpointJSON["EndpointAddress"].(string)
+			if !ok {
+				return fmt.Errorf("topic endpoint has no EndpointAddress string: %v", endpointJSON["EndpointAddress"])
+			}
 
-			uriPassword, _ := uri.User.Password()
+			parsed, err := url.Parse(addr)
+			if err != nil {
+				return err
+			}
 
-			return user == uri.User.Username() && pass == uriPassword
+			uriPassword, _ := parsed.User.Password()
+			if user != parsed.User.Username() || pass != uriPassword {
+				return fmt.Errorf("topic basic auth not yet in sync for user %q", parsed.User.Username())
+			}
+
+			// capture the matching sample for the post-wait assertions
+			uri = parsed
+			return nil
 		})
-		require.True(t, inSync)
 
 		assert.Equal(t, "kafka", uri.Scheme)
 		assert.Equal(t, "kafka.example.com:9094", uri.Host)
-		assert.Equal(t, user, uri.User.Username())
-		uriPassword, _ := uri.User.Password()
-		assert.Equal(t, pass, uriPassword)
 	})
 }
 
-// Note that .status.ARN is nil while .status.phase == Reconciling
-func cephBucketTopicReady(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBucketTopic) *cephv1.CephBucketTopic {
-	ctx := context.TODO()
-
-	var liveBt *cephv1.CephBucketTopic
-	btReady := utils.Retry(60, time.Second, fmt.Sprintf("CephBucketTopic %q is Ready", bt.Name), func() bool {
-		var err error
-		liveBt, err = k8sh.RookClientset.CephV1().CephBucketTopics(bt.Namespace).Get(ctx, bt.Name, metav1.GetOptions{})
-		if err != nil {
-			return false
-		}
-
-		if liveBt.Status == nil {
-			return false
-		}
-
-		return liveBt.Status.Phase == string(cephv1.ConditionReady)
-	})
-	require.True(t, btReady)
-	require.NotNil(t, liveBt.Status.ARN)
-
-	return liveBt
+// kvSecret returns a Secret holding a single key/value pair.
+func kvSecret(ns, name, key, value string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+		Data: map[string][]byte{
+			key: []byte(value),
+		},
+	}
 }
 
-func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) {
+// secretKeySelector returns a selector for key within the named secret.
+func secretKeySelector(name, key string) *corev1.SecretKeySelector {
+	return &corev1.SecretKeySelector{
+		LocalObjectReference: corev1.LocalObjectReference{
+			Name: name,
+		},
+		Key: key,
+	}
+}
+
+const Namespace = "test-topickafka"
+
+func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
 	var (
-		defaultName = "test-topickafka"
+		defaultName = Namespace
+		objectStore = store.ObjectStore()
 
 		ns = &corev1.Namespace{
 			ObjectMeta: metav1.ObjectMeta{
@@ -140,56 +132,12 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal
 			},
 		}
 
-		secret1 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret1",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"user-name": []byte("kafka-user1"),
-			},
-		}
+		secret1 = kvSecret(ns.Name, defaultName+"-secret1", "user-name", "kafka-user1")
+		secret2 = kvSecret(ns.Name, defaultName+"-secret2", "password", "kafka-pass2")
+		secret3 = kvSecret(ns.Name, defaultName+"-secret3", "user-name", "kafka-user3")
+		secret4 = kvSecret(ns.Name, defaultName+"-secret4", "password", "kafka-pass4")
 
-		secret2 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret2",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"password": []byte("kafka-pass2"),
-			},
-		}
-
-		secret3 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret3",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"user-name": []byte("kafka-user3"),
-			},
-		}
-
-		secret4 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret4",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"password": []byte("kafka-pass4"),
-			},
-		}
-
-		storageClass = &storagev1.StorageClass{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: defaultName,
-			},
-			Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
-			Parameters: map[string]string{
-				"objectStoreName":      objectStore.Name,
-				"objectStoreNamespace": objectStore.Namespace,
-			},
-		}
+		storageClass = fixture.StorageClass(defaultName, objectStore)
 
 		bt1 = &cephv1.CephBucketTopic{
 			ObjectMeta: metav1.ObjectMeta{
@@ -202,21 +150,11 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal
 				Persistent:           false,
 				Endpoint: cephv1.TopicEndpointSpec{
 					Kafka: &cephv1.KafkaEndpointSpec{
-						URI:      "kafka://kafka.example.com:9094",
-						AckLevel: "broker",
-						UseSSL:   false,
-						UserSecretRef: &corev1.SecretKeySelector{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: secret1.Name,
-							},
-							Key: "user-name",
-						},
-						PasswordSecretRef: &corev1.SecretKeySelector{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: secret2.Name,
-							},
-							Key: "password",
-						},
+						URI:               "kafka://kafka.example.com:9094",
+						AckLevel:          "broker",
+						UseSSL:            false,
+						UserSecretRef:     secretKeySelector(secret1.Name, "user-name"),
+						PasswordSecretRef: secretKeySelector(secret2.Name, "password"),
 					},
 				},
 			},
@@ -249,275 +187,172 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal
 			},
 		}
 
-		ctx       = context.TODO()
-		snsClient *sns.Client
+		snsClient = store.SnsClient()
+
+		btClient     = k8sh.RookClientset.CephV1().CephBucketTopics(ns.Name)
+		bnClient     = k8sh.RookClientset.CephV1().CephBucketNotifications(ns.Name)
+		obcClient    = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name)
+		obClient     = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
+		secretClient = k8sh.Clientset.CoreV1().Secrets(ns.Name)
 	)
 
 	t.Run("CephBucketTopic kafka", func(t *testing.T) {
-		if tlsEnable {
-			// Skip testing with and without TLS to reduce test time
-			t.Skip("skipping test for TLS enabled clusters")
+		ctx := t.Context()
+
+		fixture.RequireNamespace(t, k8sh, ns)
+
+		for _, secret := range []*corev1.Secret{secret1, secret2, secret3, secret4} {
+			t.Run(fmt.Sprintf("create secret %q", secret.Name), func(t *testing.T) {
+				_, err := secretClient.Create(ctx, secret, metav1.CreateOptions{})
+				require.NoError(t, err)
+			})
 		}
 
-		t.Run(fmt.Sprintf("create ns %q", ns.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret1.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Create(ctx, secret1, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret2.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(secret2.Namespace).Create(ctx, secret2, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret3.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Create(ctx, secret3, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret4.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Create(ctx, secret4, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
 		// the sc, obc, and CephBucketNotification are essentially unused for testing but are created for completeness
-		t.Run(fmt.Sprintf("create sc %q", storageClass.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.StorageV1().StorageClasses().Create(ctx, storageClass, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
+		fixture.RequireStorageClass(t, k8sh, storageClass)
 
 		t.Run(fmt.Sprintf("create obc %q", obc1.Name), func(t *testing.T) {
-			_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc1.Namespace).Create(ctx, obc1, metav1.CreateOptions{})
+			_, err := obcClient.Create(ctx, obc1, metav1.CreateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("create CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-			_, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Create(ctx, bt1, metav1.CreateOptions{})
-			require.NoError(t, err)
-
-			// user creation may be slow right after rgw start up
-			cephBucketTopicReady(t, k8sh, bt1)
+			// topic creation may be slow right after rgw start up
+			wait4.RequireCreate(ctx, t, btClient, bt1, wait4.BucketTopic, wait4.TimeoutMedium)
 		})
 
 		t.Run(fmt.Sprintf("create CephBucketNotification %q", bn1.Name), func(t *testing.T) {
-			_, err := k8sh.RookClientset.CephV1().CephBucketNotifications(bn1.Namespace).Create(ctx, bn1, metav1.CreateOptions{})
+			_, err := bnClient.Create(ctx, bn1, metav1.CreateOptions{})
 			require.NoError(t, err)
 		})
 
-		t.Run("setup sns client", func(t *testing.T) {
-			var err error
-			snsClient, err = utilsns.NewClient(objectStore, k8sh, installer, tlsEnable)
-			require.NoError(t, err)
-		})
-
-		{
-			liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-			checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret1.Data["user-name"]), string(secret2.Data["password"]))
-
-			secrets := []*corev1.Secret{secret1, secret2}
-
-			checkStatusSecrets(t, k8sh, bt1, secrets)
-		}
+		liveBt := wait4.RequireCondition(ctx, t, btClient, bt1.Name, wait4.BucketTopic, wait4.TimeoutMedium)
+		checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret1.Data["user-name"]), string(secret2.Data["password"]))
+		checkStatusSecrets(t, k8sh, bt1, secret1, secret2)
 
 		t.Run("updating referenced secrets reconciles rgw topic", func(t *testing.T) {
 			t.Run(fmt.Sprintf("update kafka auth on CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-				liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
+				liveBt, err := btClient.Get(ctx, bt1.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 
-				liveBt.Spec.Endpoint.Kafka.UserSecretRef = &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: secret3.Name,
-					},
-					Key: "user-name",
-				}
+				liveBt.Spec.Endpoint.Kafka.UserSecretRef = secretKeySelector(secret3.Name, "user-name")
+				liveBt.Spec.Endpoint.Kafka.PasswordSecretRef = secretKeySelector(secret4.Name, "password")
 
-				liveBt.Spec.Endpoint.Kafka.PasswordSecretRef = &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: secret4.Name,
-					},
-					Key: "password",
-				}
-
-				_, err = k8sh.RookClientset.CephV1().CephBucketTopics(liveBt.Namespace).Update(ctx, liveBt, metav1.UpdateOptions{})
+				_, err = btClient.Update(ctx, liveBt, metav1.UpdateOptions{})
 				require.NoError(t, err)
 			})
 
-			{
-				liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-				checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret3.Data["user-name"]), string(secret4.Data["password"]))
-
-				secrets := []*corev1.Secret{secret3, secret4}
-
-				checkStatusSecrets(t, k8sh, bt1, secrets)
-			}
+			liveBt := wait4.RequireCondition(ctx, t, btClient, bt1.Name, wait4.BucketTopic, wait4.TimeoutMedium)
+			checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret3.Data["user-name"]), string(secret4.Data["password"]))
+			checkStatusSecrets(t, k8sh, bt1, secret3, secret4)
 		})
 
 		t.Run("removing referenced secrets reconciles rgw topic", func(t *testing.T) {
 			t.Run(fmt.Sprintf("delete kafka auth on CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-				liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
+				liveBt, err := btClient.Get(ctx, bt1.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 
 				liveBt.Spec.Endpoint.Kafka.UserSecretRef = nil
 				liveBt.Spec.Endpoint.Kafka.PasswordSecretRef = nil
 
-				_, err = k8sh.RookClientset.CephV1().CephBucketTopics(liveBt.Namespace).Update(ctx, liveBt, metav1.UpdateOptions{})
+				_, err = btClient.Update(ctx, liveBt, metav1.UpdateOptions{})
 				require.NoError(t, err)
 			})
 
-			{
-				liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-				checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, "", "")
-
-				secrets := []*corev1.Secret{}
-
-				checkStatusSecrets(t, k8sh, bt1, secrets)
-			}
+			liveBt := wait4.RequireCondition(ctx, t, btClient, bt1.Name, wait4.BucketTopic, wait4.TimeoutMedium)
+			checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, "", "")
+			checkStatusSecrets(t, k8sh, bt1)
 		})
 
 		t.Run("adding new referenced secrets reconciles rgw topic", func(t *testing.T) {
 			t.Run(fmt.Sprintf("add kafka auth on CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-				liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
+				liveBt, err := btClient.Get(ctx, bt1.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 
-				liveBt.Spec.Endpoint.Kafka.UserSecretRef = &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: secret3.Name,
-					},
-					Key: "user-name",
-				}
+				liveBt.Spec.Endpoint.Kafka.UserSecretRef = secretKeySelector(secret3.Name, "user-name")
+				liveBt.Spec.Endpoint.Kafka.PasswordSecretRef = secretKeySelector(secret4.Name, "password")
 
-				liveBt.Spec.Endpoint.Kafka.PasswordSecretRef = &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: secret4.Name,
-					},
-					Key: "password",
-				}
-
-				_, err = k8sh.RookClientset.CephV1().CephBucketTopics(liveBt.Namespace).Update(ctx, liveBt, metav1.UpdateOptions{})
+				_, err = btClient.Update(ctx, liveBt, metav1.UpdateOptions{})
 				require.NoError(t, err)
 			})
 
-			{
-				liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-				checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret3.Data["user-name"]), string(secret4.Data["password"]))
-
-				secrets := []*corev1.Secret{secret3, secret4}
-
-				checkStatusSecrets(t, k8sh, bt1, secrets)
-			}
+			liveBt := wait4.RequireCondition(ctx, t, btClient, bt1.Name, wait4.BucketTopic, wait4.TimeoutMedium)
+			checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret3.Data["user-name"]), string(secret4.Data["password"]))
+			checkStatusSecrets(t, k8sh, bt1, secret3, secret4)
 		})
 
 		t.Run("updating secrets reconciles rgw topic", func(t *testing.T) {
 			t.Run(fmt.Sprintf("update secret %q data", secret3.Name), func(t *testing.T) {
-				liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Get(ctx, secret3.Name, metav1.GetOptions{})
+				liveSecret, err := secretClient.Get(ctx, secret3.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 
 				liveSecret.Data = map[string][]byte{
 					"user-name": []byte("kafka-user3-updated"),
 				}
 
-				_, err = k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Update(ctx, liveSecret, metav1.UpdateOptions{})
+				_, err = secretClient.Update(ctx, liveSecret, metav1.UpdateOptions{})
 				require.NoError(t, err)
 			})
 
 			t.Run(fmt.Sprintf("update secret %q data", secret4.Name), func(t *testing.T) {
-				liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Get(ctx, secret4.Name, metav1.GetOptions{})
+				liveSecret, err := secretClient.Get(ctx, secret4.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 
 				liveSecret.Data = map[string][]byte{
 					"password": []byte("kafka-pass4-updated"),
 				}
 
-				_, err = k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Update(ctx, liveSecret, metav1.UpdateOptions{})
+				_, err = secretClient.Update(ctx, liveSecret, metav1.UpdateOptions{})
 				require.NoError(t, err)
 			})
 
-			{
-				liveBt := cephBucketTopicReady(t, k8sh, bt1)
+			liveBt := wait4.RequireCondition(ctx, t, btClient, bt1.Name, wait4.BucketTopic, wait4.TimeoutMedium)
 
-				liveSecret3, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Get(ctx, secret3.Name, metav1.GetOptions{})
-				require.NoError(t, err)
+			liveSecret3, err := secretClient.Get(ctx, secret3.Name, metav1.GetOptions{})
+			require.NoError(t, err)
 
-				liveSecret4, err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Get(ctx, secret4.Name, metav1.GetOptions{})
-				require.NoError(t, err)
+			liveSecret4, err := secretClient.Get(ctx, secret4.Name, metav1.GetOptions{})
+			require.NoError(t, err)
 
-				checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(liveSecret3.Data["user-name"]), string(liveSecret4.Data["password"]))
-
-				secrets := []*corev1.Secret{liveSecret3, liveSecret4}
-
-				checkStatusSecrets(t, k8sh, bt1, secrets)
-			}
+			checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(liveSecret3.Data["user-name"]), string(liveSecret4.Data["password"]))
+			checkStatusSecrets(t, k8sh, bt1, liveSecret3, liveSecret4)
 		})
 
 		t.Run("deleting a referenced secret triggers a reconcile, which fails", func(t *testing.T) {
 			t.Run(fmt.Sprintf("delete secret %q", secret4.Name), func(t *testing.T) {
-				err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Delete(ctx, secret4.Name, metav1.DeleteOptions{})
+				err := secretClient.Delete(ctx, secret4.Name, metav1.DeleteOptions{})
 				require.NoError(t, err)
 			})
 
 			t.Run(fmt.Sprintf("CephBucketTopic %q has phase ReconcileFailed", bt1.Name), func(t *testing.T) {
-				btReady := utils.Retry(40, time.Second, "CephBucketTopic is ReconcileFailed", func() bool {
-					liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-					if err != nil {
-						return false
-					}
-
-					if liveBt.Status == nil {
-						return false
-					}
-
-					return liveBt.Status.Phase == string(cephv1.ReconcileFailed)
-				})
-				require.True(t, btReady)
+				wait4.RequireCondition(ctx, t, btClient, bt1.Name, wait4.BucketTopicPhase(string(cephv1.ReconcileFailed)), wait4.TimeoutShort)
 			})
 
 			t.Run(fmt.Sprintf("create secret %q", secret4.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Create(ctx, secret4, metav1.CreateOptions{})
+				_, err := secretClient.Create(ctx, secret4, metav1.CreateOptions{})
 				require.NoError(t, err)
 			})
 
-			cephBucketTopicReady(t, k8sh, bt1)
+			wait4.RequireCondition(ctx, t, btClient, bt1.Name, wait4.BucketTopic, wait4.TimeoutMedium)
 		})
 
 		t.Run(fmt.Sprintf("delete CephBucketNotification %q", bn1.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephBucketNotifications(bn1.Namespace).Delete(ctx, bn1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "CephBucketNotification is absent", func() bool {
-				_, err := k8sh.RookClientset.CephV1().CephBucketNotifications(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			wait4.AssertDelete(ctx, t, bnClient, bn1.Name, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("no CephBucketNotification(s) in ns %q", ns.Name), func(t *testing.T) {
-			list, err := k8sh.RookClientset.CephV1().CephBucketNotifications(ns.Name).List(ctx, metav1.ListOptions{})
+			list, err := bnClient.List(ctx, metav1.ListOptions{})
 			require.NoError(t, err)
 
 			assert.Len(t, list.Items, 0)
 		})
 
 		t.Run(fmt.Sprintf("delete CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Delete(ctx, bt1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "CephBucketTopic is absent", func() bool {
-				_, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			wait4.AssertDelete(ctx, t, btClient, bt1.Name, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("no CephBucketTopic(s) in ns %q", ns.Name), func(t *testing.T) {
-			list, err := k8sh.RookClientset.CephV1().CephBucketTopics(ns.Name).List(ctx, metav1.ListOptions{})
+			list, err := btClient.List(ctx, metav1.ListOptions{})
 			require.NoError(t, err)
 
 			assert.Len(t, list.Items, 0)
@@ -525,29 +360,17 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal
 
 		t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) {
 			// lookup ob name
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc1.Namespace).Get(ctx, obc1.Name, metav1.GetOptions{})
+			liveObc, err := obcClient.Get(ctx, obc1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 			obName := liveObc.Spec.ObjectBucketName
 
-			// delete obc
-			err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc1.Namespace).Delete(ctx, obc1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "OBC is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc1.Namespace).Get(ctx, obc1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-
-			absent = utils.Retry(40, time.Second, "OB is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			// delete obc; the backing ob is garbage-collected by the provisioner
+			wait4.AssertDelete(ctx, t, obcClient, obc1.Name, wait4.TimeoutShort)
+			wait4.AssertAbsent(ctx, t, obClient, obName, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) {
-			list, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).List(ctx, metav1.ListOptions{})
+			list, err := obcClient.List(ctx, metav1.ListOptions{})
 			require.NoError(t, err)
 
 			assert.Len(t, list.Items, 0)
@@ -555,61 +378,51 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal
 
 		t.Run("CephBucketTopic deletion did not remove any secrets", func(t *testing.T) {
 			t.Run(fmt.Sprintf("secret %q still exists", secret1.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Get(ctx, secret1.Name, metav1.GetOptions{})
+				_, err := secretClient.Get(ctx, secret1.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 			})
 
 			t.Run(fmt.Sprintf("secret %q still exists", secret2.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret2.Namespace).Get(ctx, secret2.Name, metav1.GetOptions{})
+				_, err := secretClient.Get(ctx, secret2.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 			})
 
 			t.Run(fmt.Sprintf("secret %q still exists", secret3.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Get(ctx, secret3.Name, metav1.GetOptions{})
+				_, err := secretClient.Get(ctx, secret3.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 			})
 
 			t.Run(fmt.Sprintf("secret %q still exists", secret4.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Get(ctx, secret4.Name, metav1.GetOptions{})
+				_, err := secretClient.Get(ctx, secret4.Name, metav1.GetOptions{})
 				require.NoError(t, err)
 			})
 		})
 
-		t.Run(fmt.Sprintf("delete sc %q", storageClass.Name), func(t *testing.T) {
-			err := k8sh.Clientset.StorageV1().StorageClasses().Delete(ctx, storageClass.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
 		t.Run(fmt.Sprintf("delete secret %q", secret4.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Delete(ctx, secret4.Name, metav1.DeleteOptions{})
+			err := secretClient.Delete(ctx, secret4.Name, metav1.DeleteOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("delete secret %q", secret3.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Delete(ctx, secret3.Name, metav1.DeleteOptions{})
+			err := secretClient.Delete(ctx, secret3.Name, metav1.DeleteOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("delete secret %q", secret2.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Delete(ctx, secret2.Name, metav1.DeleteOptions{})
+			err := secretClient.Delete(ctx, secret2.Name, metav1.DeleteOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("delete secret %q", secret1.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Delete(ctx, secret1.Name, metav1.DeleteOptions{})
+			err := secretClient.Delete(ctx, secret1.Name, metav1.DeleteOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("no secrets in ns %q", ns.Name), func(t *testing.T) {
-			secrets, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).List(ctx, metav1.ListOptions{})
+			secrets, err := secretClient.List(ctx, metav1.ListOptions{})
 			require.NoError(t, err)
 
 			assert.Len(t, secrets.Items, 0)
 		})
-
-		t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
 	})
 }

tests/integration/object/user/caps/caps.go

@@ -20,10 +20,8 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/ceph/go-ceph/rgw/admin"
-	"github.com/coreos/pkg/capnslog"
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,14 +29,19 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
-	utiladmin "github.com/rook/rook/tests/integration/object/util/admin"
+	"github.com/rook/rook/tests/integration/object/util/fixture"
+	"github.com/rook/rook/tests/integration/object/util/sharedstore"
+	"github.com/rook/rook/tests/integration/object/util/wait4"
 )
 
-func TestObjectStoreUserCaps(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) {
+const Namespace = "test-usercaps"
+
+func TestObjectStoreUserCaps(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
 	var (
-		defaultName = "test-usercaps"
+		defaultName = Namespace
+		objectStore = store.ObjectStore()
+		adminClient = store.AdminClient()
 
 		ns = &corev1.Namespace{
 			ObjectMeta: metav1.ObjectMeta{
@@ -56,47 +59,18 @@ func TestObjectStoreUserCaps(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 				ClusterNamespace: objectStore.Namespace,
 			},
 		}
+
+		osuClient = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name)
 	)
 
 	t.Run("ObjectStoreUser caps", func(t *testing.T) {
-		if tlsEnable {
-			// Skip testing with and without TLS to reduce test time
-			t.Skip("skipping test for TLS enabled clusters")
-		}
+		ctx := t.Context()
 
-		var adminClient *admin.API
-		ctx := context.TODO()
-
-		t.Run(fmt.Sprintf("create ns %q", ns.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
+		fixture.RequireNamespace(t, k8sh, ns)
 
 		t.Run(fmt.Sprintf("create CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Create(ctx, &osu1, metav1.CreateOptions{})
-			require.NoError(t, err)
-
 			// user creation may be slow right after rgw start up
-			osuReady := utils.Retry(120, time.Second, "CephObjectStoreUser is Ready", func() bool {
-				liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				if liveOsu.Status == nil {
-					return false
-				}
-
-				return liveOsu.Status.Phase == string(cephv1.ConditionReady)
-			})
-			require.True(t, osuReady)
-		})
-
-		// wait for cosu user to be ready so we know rgw admin api is ready
-		t.Run("setup rgw admin api client", func(t *testing.T) {
-			var err error
-			adminClient, err = utiladmin.NewAdminClient(objectStore, installer, k8sh, tlsEnable)
-			require.NoError(t, err)
+			wait4.RequireCreate(ctx, t, osuClient, &osu1, wait4.ObjectStoreUser, wait4.TimeoutLong)
 		})
 
 		t.Run(fmt.Sprintf("verify caps on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
@@ -106,7 +80,7 @@ func TestObjectStoreUserCaps(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 		})
 
 		t.Run(fmt.Sprintf("update caps on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveOsu.Spec.Capabilities = &cephv1.ObjectUserCapSpec{
@@ -114,18 +88,18 @@ func TestObjectStoreUserCaps(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 				Usage:   "read",
 			}
 
-			_, err = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Update(ctx, liveOsu, metav1.UpdateOptions{})
+			_, err = osuClient.Update(ctx, liveOsu, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("verify caps on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			converged := utils.Retry(30, time.Second, "caps updated", func() bool {
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, "rgw user caps match spec", func(ctx context.Context) error {
 				liveUser, err := adminClient.GetUser(ctx, admin.User{ID: osu1.Name})
 				if err != nil {
-					return false
+					return err
 				}
 
-				return cmp.Equal(liveUser.Caps, []admin.UserCapSpec{
+				if !cmp.Equal(liveUser.Caps, []admin.UserCapSpec{
 					{
 						Type: "buckets",
 						Perm: "*",
@@ -134,56 +108,46 @@ func TestObjectStoreUserCaps(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 						Type: "usage",
 						Perm: "read",
 					},
-				})
+				}) {
+					return fmt.Errorf("caps not yet in sync: %+v", liveUser.Caps)
+				}
+				return nil
 			})
-
-			assert.True(t, converged)
 		})
 
 		t.Run(fmt.Sprintf("remove caps on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveOsu.Spec.Capabilities = nil
 
-			_, err = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Update(ctx, liveOsu, metav1.UpdateOptions{})
+			_, err = osuClient.Update(ctx, liveOsu, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("verify caps on CephObjectStoreUser %q returns to default", osu1.Name), func(t *testing.T) {
-			converged := utils.Retry(30, time.Second, "caps updated", func() bool {
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, "rgw user caps are empty", func(ctx context.Context) error {
 				liveUser, err := adminClient.GetUser(ctx, admin.User{ID: osu1.Name})
 				if err != nil {
-					return false
+					return err
 				}
 
-				return len(liveUser.Caps) == 0
+				if len(liveUser.Caps) != 0 {
+					return fmt.Errorf("caps not yet empty: %+v", liveUser.Caps)
+				}
+				return nil
 			})
-
-			assert.True(t, converged)
 		})
 
 		t.Run(fmt.Sprintf("delete CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Delete(ctx, osu1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "CephObjectStoreUser is absent", func() bool {
-				_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			wait4.AssertDelete(ctx, t, osuClient, osu1.Name, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("no CephObjectStoreUsers in ns %q", ns.Name), func(t *testing.T) {
-			osus, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).List(ctx, metav1.ListOptions{})
+			osus, err := osuClient.List(ctx, metav1.ListOptions{})
 			require.NoError(t, err)
 
 			assert.Len(t, osus.Items, 0)
 		})
-
-		t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
 	})
 }

tests/integration/object/user/keys/keys.go

@@ -20,19 +20,19 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/ceph/go-ceph/rgw/admin"
-	"github.com/coreos/pkg/capnslog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
-	utiladmin "github.com/rook/rook/tests/integration/object/util/admin"
+	"github.com/rook/rook/tests/integration/object/util/fixture"
+	"github.com/rook/rook/tests/integration/object/util/secrets"
+	"github.com/rook/rook/tests/integration/object/util/sharedstore"
+	"github.com/rook/rook/tests/integration/object/util/wait4"
 )
 
 // generate the secret name the operator is expected to generate for the CephObjectStoreUser
@@ -40,6 +40,40 @@ func generateObjectStoreUserSecretName(osu cephv1.CephObjectStoreUser) string {
 	return "rook-ceph-object-user-" + osu.Spec.Store + "-" + osu.Name
 }
 
+// awsKeySecret returns a Secret holding an s3 access/secret key pair under the
+// conventional AWS env var names.
+func awsKeySecret(ns, name, accessKey, secretKey string) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+		Data: map[string][]byte{
+			"AWS_ACCESS_KEY_ID":     []byte(accessKey),
+			"AWS_SECRET_ACCESS_KEY": []byte(secretKey),
+		},
+	}
+}
+
+// objectUserKey returns an ObjectUserKey referencing the access/secret key
+// pair stored in the named awsKeySecret secret.
+func objectUserKey(secretName string) cephv1.ObjectUserKey {
+	return cephv1.ObjectUserKey{
+		AccessKeyRef: &corev1.SecretKeySelector{
+			LocalObjectReference: corev1.LocalObjectReference{
+				Name: secretName,
+			},
+			Key: "AWS_ACCESS_KEY_ID",
+		},
+		SecretKeyRef: &corev1.SecretKeySelector{
+			LocalObjectReference: corev1.LocalObjectReference{
+				Name: secretName,
+			},
+			Key: "AWS_SECRET_ACCESS_KEY",
+		},
+	}
+}
+
 // find a UserKeySpec by AccessKey value
 func findUserKeySpec(keys []admin.UserKeySpec, accessKey string) (admin.UserKeySpec, error) {
 	for _, k := range keys {
@@ -50,71 +84,63 @@ func findUserKeySpec(keys []admin.UserKeySpec, accessKey string) (admin.UserKeyS
 	return admin.UserKeySpec{}, fmt.Errorf("UserKeySpec with AccessKey %q not found", accessKey)
 }
 
-func checkStatusKeys(t *testing.T, k8sh *utils.K8sHelper, osu cephv1.CephObjectStoreUser, expectedSecrets []*corev1.Secret) {
-	t.Run(fmt.Sprintf("cephObjectStoreUser %q has .status.keys set", osu.Name), func(t *testing.T) {
-		ctx := context.TODO()
-
-		liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(osu.Namespace).Get(ctx, osu.Name, metav1.GetOptions{})
-		require.NoError(t, err)
-
-		require.NotNil(t, liveOsu.Status)
-		assert.Len(t, liveOsu.Status.Keys, len(expectedSecrets))
-
-		for _, secret := range expectedSecrets {
-			secretRef, err := func(secretName string, keys []cephv1.SecretReference) (cephv1.SecretReference, error) {
-				for _, secretRef := range keys {
-					if secretRef.Name == secretName {
-						return secretRef, nil
-					}
-				}
-				return cephv1.SecretReference{}, fmt.Errorf("secretReference for secret %q not found in CephObjectStoreUser.status.keys", secret.Name)
-			}(secret.Name, liveOsu.Status.Keys)
-			require.NoError(t, err)
-
-			// fetch the live secret for UID and ResourceVersion
-			liveSecret, err := k8sh.Clientset.CoreV1().Secrets(osu.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
-			require.NoError(t, err)
-
-			assert.Equal(t, liveSecret.Name, secretRef.Name)
-			assert.Equal(t, liveSecret.Namespace, secretRef.Namespace)
-			assert.Equal(t, liveSecret.UID, secretRef.UID)
-			assert.Equal(t, liveSecret.ResourceVersion, secretRef.ResourceVersion)
-		}
-	})
+func checkStatusKeys(t *testing.T, k8sh *utils.K8sHelper, osu cephv1.CephObjectStoreUser, expected ...*corev1.Secret) {
+	secrets.RequireStatusRefs(t, k8sh,
+		k8sh.RookClientset.CephV1().CephObjectStoreUsers(osu.Namespace), osu.Name,
+		fmt.Sprintf("cephObjectStoreUser %q has .status.keys set", osu.Name),
+		func(u *cephv1.CephObjectStoreUser) []cephv1.SecretReference {
+			if u.Status == nil {
+				return nil
+			}
+			return u.Status.Keys
+		},
+		expected...)
 }
 
-func checkRgwUserKeys(t *testing.T, adminClient *admin.API, osu cephv1.CephObjectStoreUser, expectedSecrets []*corev1.Secret, accessKeyName, secretKeyName string) {
-	t.Run(fmt.Sprintf("rgw user %q has keys set", osu.Name), func(t *testing.T) {
-		ctx := context.TODO()
+// requireRgwUserKeys verifies every expected key on the rgw user — reporting
+// all divergences rather than stopping at the first — and then aborts the
+// caller if any check failed: the steps that follow assume the keys are in
+// sync, so continuing would only cascade noise.
+func requireRgwUserKeys(t *testing.T, adminClient *admin.API, osu cephv1.CephObjectStoreUser, accessKeyName, secretKeyName string, expected ...*corev1.Secret) {
+	t.Helper()
 
-		for _, secret := range expectedSecrets {
-			var keySpec admin.UserKeySpec
+	if !t.Run(fmt.Sprintf("rgw user %q has keys set", osu.Name), func(t *testing.T) {
+		ctx := t.Context()
 
+		for _, secret := range expected {
 			// assume that the .Phase doesn't change when updating keys
-			inSync := utils.Retry(40, time.Second, fmt.Sprintf("CephObjectStoreUser has key in sync with secret %q ", secret.Name), func() bool {
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, fmt.Sprintf("rgw user key in sync with secret %q", secret.Name), func(ctx context.Context) error {
 				liveUser, err := adminClient.GetUser(ctx, admin.User{ID: osu.Name})
 				if err != nil {
-					return false
+					return err
 				}
-				keySpec, err = findUserKeySpec(liveUser.Keys, string(secret.Data[accessKeyName]))
-				return err == nil
+				keySpec, err := findUserKeySpec(liveUser.Keys, string(secret.Data[accessKeyName]))
+				if err != nil {
+					return err
+				}
+				if keySpec.SecretKey != string(secret.Data[secretKeyName]) {
+					return fmt.Errorf("secret key for %q not yet in sync", secret.Name)
+				}
+				return nil
 			})
-			require.True(t, inSync)
-
-			assert.Equal(t, string(secret.Data[accessKeyName]), keySpec.AccessKey)
-			assert.Equal(t, string(secret.Data[secretKeyName]), keySpec.SecretKey)
 		}
 
 		// check that no extra keys are present
 		liveUser, err := adminClient.GetUser(ctx, admin.User{ID: osu.Name})
 		require.NoError(t, err)
-		assert.Equal(t, len(expectedSecrets), len(liveUser.Keys))
-	})
+		assert.Equal(t, len(expected), len(liveUser.Keys))
+	}) {
+		t.FailNow()
+	}
 }
 
-func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) {
+const Namespace = "test-userkeys"
+
+func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
 	var (
-		defaultName = "test-userkeys"
+		defaultName = Namespace
+		objectStore = store.ObjectStore()
+		adminClient = store.AdminClient()
 
 		ns = &corev1.Namespace{
 			ObjectMeta: metav1.ObjectMeta{
@@ -122,60 +148,11 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 			},
 		}
 
-		secret1 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret1",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"AWS_ACCESS_KEY_ID":     []byte("P1ANF2BP4K2LPGOR5SBB"),
-				"AWS_SECRET_ACCESS_KEY": []byte("yfCHPhhOed0vJkqZsGEODyJrdKqHD09OMTWCnwjX"),
-			},
-		}
-
-		secret2 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret2",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"AWS_ACCESS_KEY_ID":     []byte("2I6RPUTQFMNCSYEXZ6VM"),
-				"AWS_SECRET_ACCESS_KEY": []byte("uY066SWPfaOVlDeYc7GJyOTfkDejyDdXrqehS6wx"),
-			},
-		}
-
-		secret3 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret3",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"AWS_ACCESS_KEY_ID":     []byte("J4D0P20F3EDR51OSND7Y"),
-				"AWS_SECRET_ACCESS_KEY": []byte("jn89OpkXNoDlIHVVQ23mE2DZgPmuDK3WVH5ExOvQ"),
-			},
-		}
-
-		secret4 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret4",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"AWS_ACCESS_KEY_ID":     []byte("MPZX7DG5WJWQ6VPCSLYT"),
-				"AWS_SECRET_ACCESS_KEY": []byte("phh7DIxnLPeSD2V6FUouhmnWrKlKRD5dBykyXozX"),
-			},
-		}
-
-		secret5 = &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      defaultName + "-secret5",
-				Namespace: ns.Name,
-			},
-			Data: map[string][]byte{
-				"AWS_ACCESS_KEY_ID":     []byte("7TNSSANCO5KXK23IPT91"),
-				"AWS_SECRET_ACCESS_KEY": []byte("HksEDf0hEh3PtTvl7s9x6CyXfkWuY8eAMYAcvH5l"),
-			},
-		}
+		secret1 = awsKeySecret(ns.Name, defaultName+"-secret1", "P1ANF2BP4K2LPGOR5SBB", "yfCHPhhOed0vJkqZsGEODyJrdKqHD09OMTWCnwjX")
+		secret2 = awsKeySecret(ns.Name, defaultName+"-secret2", "2I6RPUTQFMNCSYEXZ6VM", "uY066SWPfaOVlDeYc7GJyOTfkDejyDdXrqehS6wx")
+		secret3 = awsKeySecret(ns.Name, defaultName+"-secret3", "J4D0P20F3EDR51OSND7Y", "jn89OpkXNoDlIHVVQ23mE2DZgPmuDK3WVH5ExOvQ")
+		secret4 = awsKeySecret(ns.Name, defaultName+"-secret4", "MPZX7DG5WJWQ6VPCSLYT", "phh7DIxnLPeSD2V6FUouhmnWrKlKRD5dBykyXozX")
+		secret5 = awsKeySecret(ns.Name, defaultName+"-secret5", "7TNSSANCO5KXK23IPT91", "HksEDf0hEh3PtTvl7s9x6CyXfkWuY8eAMYAcvH5l")
 
 		osu1 = cephv1.CephObjectStoreUser{
 			ObjectMeta: metav1.ObjectMeta{
@@ -186,208 +163,85 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 				Store:            objectStore.Name,
 				ClusterNamespace: objectStore.Namespace,
 				Keys: []cephv1.ObjectUserKey{
-					{
-						AccessKeyRef: &corev1.SecretKeySelector{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: secret1.Name,
-							},
-							Key: "AWS_ACCESS_KEY_ID",
-						},
-						SecretKeyRef: &corev1.SecretKeySelector{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: secret1.Name,
-							},
-							Key: "AWS_SECRET_ACCESS_KEY",
-						},
-					},
-					{
-						AccessKeyRef: &corev1.SecretKeySelector{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: secret2.Name,
-							},
-							Key: "AWS_ACCESS_KEY_ID",
-						},
-						SecretKeyRef: &corev1.SecretKeySelector{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: secret2.Name,
-							},
-							Key: "AWS_SECRET_ACCESS_KEY",
-						},
-					},
-					{
-						AccessKeyRef: &corev1.SecretKeySelector{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: secret3.Name,
-							},
-							Key: "AWS_ACCESS_KEY_ID",
-						},
-						SecretKeyRef: &corev1.SecretKeySelector{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: secret3.Name,
-							},
-							Key: "AWS_SECRET_ACCESS_KEY",
-						},
-					},
+					objectUserKey(secret1.Name),
+					objectUserKey(secret2.Name),
+					objectUserKey(secret3.Name),
 				},
 			},
 		}
+
+		osuClient    = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name)
+		secretClient = k8sh.Clientset.CoreV1().Secrets(ns.Name)
 	)
 	t.Run("ObjectStoreUser keys", func(t *testing.T) {
-		if tlsEnable {
-			// Skip testing with and without TLS to reduce test time
-			t.Skip("skipping test for TLS enabled clusters")
+		ctx := t.Context()
+
+		fixture.RequireNamespace(t, k8sh, ns)
+
+		for _, secret := range []*corev1.Secret{secret1, secret2, secret3, secret4, secret5} {
+			t.Run(fmt.Sprintf("create secret %q", secret.Name), func(t *testing.T) {
+				_, err := secretClient.Create(ctx, secret, metav1.CreateOptions{})
+				require.NoError(t, err)
+			})
 		}
 
-		var adminClient *admin.API
-		ctx := context.TODO()
-
-		t.Run(fmt.Sprintf("create ns %q", ns.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret1.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Create(ctx, secret1, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret2.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Create(ctx, secret2, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret3.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Create(ctx, secret3, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret4.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Create(ctx, secret4, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret5.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Create(ctx, secret5, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
 		t.Run(fmt.Sprintf("create CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Create(ctx, &osu1, metav1.CreateOptions{})
-			require.NoError(t, err)
-
 			// user creation may be slow right after rgw start up
-			osuReady := utils.Retry(120, time.Second, "CephObjectStoreUser is Ready", func() bool {
-				liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				if liveOsu.Status == nil {
-					return false
-				}
-
-				return liveOsu.Status.Phase == string(cephv1.ConditionReady)
-			})
-			require.True(t, osuReady)
+			wait4.RequireCreate(ctx, t, osuClient, &osu1, wait4.ObjectStoreUser, wait4.TimeoutLong)
 		})
 
-		t.Run("setup rgw admin api client", func(t *testing.T) {
-			var err error
-			adminClient, err = utiladmin.NewAdminClient(objectStore, installer, k8sh, tlsEnable)
-			require.NoError(t, err)
-		})
-
-		{
-			secrets := []*corev1.Secret{secret1, secret2, secret3}
-
-			checkRgwUserKeys(t, adminClient, osu1, secrets, "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
-			checkStatusKeys(t, k8sh, osu1, secrets)
-		}
+		requireRgwUserKeys(t, adminClient, osu1, "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", secret1, secret2, secret3)
+		checkStatusKeys(t, k8sh, osu1, secret1, secret2, secret3)
 
 		t.Run(fmt.Sprintf("update keys on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveOsu.Spec.Keys = []cephv1.ObjectUserKey{
-				{
-					AccessKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret4.Name,
-						},
-						Key: "AWS_ACCESS_KEY_ID",
-					},
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret4.Name,
-						},
-						Key: "AWS_SECRET_ACCESS_KEY",
-					},
-				},
-				{
-					AccessKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret5.Name,
-						},
-						Key: "AWS_ACCESS_KEY_ID",
-					},
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret5.Name,
-						},
-						Key: "AWS_SECRET_ACCESS_KEY",
-					},
-				},
+				objectUserKey(secret4.Name),
+				objectUserKey(secret5.Name),
 			}
 
-			_, err = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Update(ctx, liveOsu, metav1.UpdateOptions{})
+			_, err = osuClient.Update(ctx, liveOsu, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
-		{
-			secrets := []*corev1.Secret{secret4, secret5}
-
-			checkRgwUserKeys(t, adminClient, osu1, secrets, "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
-			checkStatusKeys(t, k8sh, osu1, secrets)
-		}
+		requireRgwUserKeys(t, adminClient, osu1, "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", secret4, secret5)
+		checkStatusKeys(t, k8sh, osu1, secret4, secret5)
 
 		// test transition from explicit keys -> automatic secret creation
 
 		// when all explicit keys are removed from CephObjectStoreUser, one should
 		// be left in place and the operator should [still] create a k8s secret for it
 		t.Run(fmt.Sprintf("remove all keys set on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveOsu.Spec.Keys = nil
 
-			_, err = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Update(ctx, liveOsu, metav1.UpdateOptions{})
+			_, err = osuClient.Update(ctx, liveOsu, metav1.UpdateOptions{})
 			require.NoError(t, err)
 
-			// wait for the number of keys to drop to 1
-			inSync := utils.Retry(40, time.Second, "CephObjectStoreUser has 1 key", func() bool {
+			wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "rgw user has exactly 1 key", func(ctx context.Context) error {
 				liveUser, err := adminClient.GetUser(ctx, admin.User{ID: osu1.Name})
 				if err != nil {
-					return false
+					return err
 				}
-				return len(liveUser.Keys) == 1
+				if len(liveUser.Keys) != 1 {
+					return fmt.Errorf("rgw user has %d keys, want 1", len(liveUser.Keys))
+				}
+				return nil
 			})
-			require.True(t, inSync)
 		})
 
-		// keys updated on user
-		{
-			// fetch automatic secret as it should be the only key set on the rgw user
-			secretName := generateObjectStoreUserSecretName(osu1)
-			liveSecret, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Get(ctx, secretName, metav1.GetOptions{})
-			require.NoError(t, err)
+		// fetch the automatic secret; it should hold the only key set on the rgw user
+		autoSecret, err := secretClient.Get(ctx, generateObjectStoreUserSecretName(osu1), metav1.GetOptions{})
+		require.NoError(t, err)
 
-			secrets := []*corev1.Secret{liveSecret}
-
-			checkRgwUserKeys(t, adminClient, osu1, secrets, "AccessKey", "SecretKey")
-		}
+		requireRgwUserKeys(t, adminClient, osu1, "AccessKey", "SecretKey", autoSecret)
 
 		t.Run(fmt.Sprintf("cephObjectStoreUser %q .status.keys is unset", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			require.NotNil(t, liveOsu.Status)
@@ -396,96 +250,27 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 
 		// test transition automatic secret creation -> explicit keys
 		t.Run(fmt.Sprintf("add keys to CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveOsu.Spec.Keys = []cephv1.ObjectUserKey{
-				{
-					AccessKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret1.Name,
-						},
-						Key: "AWS_ACCESS_KEY_ID",
-					},
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret1.Name,
-						},
-						Key: "AWS_SECRET_ACCESS_KEY",
-					},
-				},
-				{
-					AccessKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret2.Name,
-						},
-						Key: "AWS_ACCESS_KEY_ID",
-					},
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret2.Name,
-						},
-						Key: "AWS_SECRET_ACCESS_KEY",
-					},
-				},
-				{
-					AccessKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret3.Name,
-						},
-						Key: "AWS_ACCESS_KEY_ID",
-					},
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret3.Name,
-						},
-						Key: "AWS_SECRET_ACCESS_KEY",
-					},
-				},
-				{
-					AccessKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret4.Name,
-						},
-						Key: "AWS_ACCESS_KEY_ID",
-					},
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret4.Name,
-						},
-						Key: "AWS_SECRET_ACCESS_KEY",
-					},
-				},
-				{
-					AccessKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret5.Name,
-						},
-						Key: "AWS_ACCESS_KEY_ID",
-					},
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret5.Name,
-						},
-						Key: "AWS_SECRET_ACCESS_KEY",
-					},
-				},
+				objectUserKey(secret1.Name),
+				objectUserKey(secret2.Name),
+				objectUserKey(secret3.Name),
+				objectUserKey(secret4.Name),
+				objectUserKey(secret5.Name),
 			}
 
-			_, err = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Update(ctx, liveOsu, metav1.UpdateOptions{})
+			_, err = osuClient.Update(ctx, liveOsu, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
-		{
-			secrets := []*corev1.Secret{secret1, secret2, secret3, secret4, secret5}
-
-			checkRgwUserKeys(t, adminClient, osu1, secrets, "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
-			checkStatusKeys(t, k8sh, osu1, secrets)
-		}
+		requireRgwUserKeys(t, adminClient, osu1, "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", secret1, secret2, secret3, secret4, secret5)
+		checkStatusKeys(t, k8sh, osu1, secret1, secret2, secret3, secret4, secret5)
 
 		// updating a secret already referenced by a CephObjectStoreUser should trigger a reconcile
 		t.Run(fmt.Sprintf("update secret %q data", secret1.Name), func(t *testing.T) {
-			liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Get(ctx, secret1.Name, metav1.GetOptions{})
+			liveSecret, err := secretClient.Get(ctx, secret1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveSecret.Data = map[string][]byte{
@@ -493,59 +278,34 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 				"AWS_SECRET_ACCESS_KEY": []byte("bar"),
 			}
 
-			_, err = k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Update(ctx, liveSecret, metav1.UpdateOptions{})
+			_, err = secretClient.Update(ctx, liveSecret, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
-		{
-			// fetch secret that we modified
-			liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Get(ctx, secret1.Name, metav1.GetOptions{})
-			require.NoError(t, err)
+		updatedSecret1, err := secretClient.Get(ctx, secret1.Name, metav1.GetOptions{})
+		require.NoError(t, err)
 
-			secrets := []*corev1.Secret{liveSecret, secret2, secret3, secret4, secret5}
-
-			checkRgwUserKeys(t, adminClient, osu1, secrets, "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
-			checkStatusKeys(t, k8sh, osu1, secrets)
-		}
+		requireRgwUserKeys(t, adminClient, osu1, "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", updatedSecret1, secret2, secret3, secret4, secret5)
+		checkStatusKeys(t, k8sh, osu1, updatedSecret1, secret2, secret3, secret4, secret5)
 
 		// deleting a secret referenced by a CephObjectStoreUser should trigger a reconcile (which fails)
 		t.Run(fmt.Sprintf("delete secret %q", secret1.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Delete(ctx, secret1.Name, metav1.DeleteOptions{})
+			err := secretClient.Delete(ctx, secret1.Name, metav1.DeleteOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("CephObjectStoreUser %q has phase ReconcileFailed", osu1.Name), func(t *testing.T) {
-			// user creation may be slow right after rgw start up
-			osuReady := utils.Retry(40, time.Second, "CephObjectStoreUser is ReconcileFailed", func() bool {
-				liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				if liveOsu.Status == nil {
-					return false
-				}
-
-				return liveOsu.Status.Phase == string(cephv1.ReconcileFailed)
-			})
-			require.True(t, osuReady)
+			wait4.RequireCondition(ctx, t, osuClient, osu1.Name, wait4.ObjectStoreUserPhase(string(cephv1.ReconcileFailed)), wait4.TimeoutShort)
 		})
 
 		t.Run("missing referenced secret should not block CephObjectStoreUser deletion", func(t *testing.T) {
 			t.Run(fmt.Sprintf("delete CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-				err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Delete(ctx, osu1.Name, metav1.DeleteOptions{})
-				require.NoError(t, err)
-
-				absent := utils.Retry(40, time.Second, "CephObjectStoreUser is absent", func() bool {
-					_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-					return err != nil
-				})
-				assert.True(t, absent)
+				wait4.AssertDelete(ctx, t, osuClient, osu1.Name, wait4.TimeoutShort)
 			})
 		})
 
 		t.Run(fmt.Sprintf("no CephObjectStoreUsers in ns %q", ns.Name), func(t *testing.T) {
-			osus, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).List(ctx, metav1.ListOptions{})
+			osus, err := osuClient.List(ctx, metav1.ListOptions{})
 			require.NoError(t, err)
 
 			assert.Len(t, osus.Items, 0)
@@ -558,39 +318,25 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins
 
 		// check that CephObjectStoreUser removal did not delete any secret resources
 
-		{
-			secrets := []*corev1.Secret{
-				// secret1 was deleted in a previous step
-				secret2,
-				secret3,
-				secret4,
-				secret5,
-			}
+		// secret1 was deleted in a previous step
+		for _, secret := range []*corev1.Secret{secret2, secret3, secret4, secret5} {
+			t.Run(fmt.Sprintf("secret %q still exists", secret.Name), func(t *testing.T) {
+				_, err := secretClient.Get(ctx, secret.Name, metav1.GetOptions{})
+				require.NoError(t, err)
+			})
 
-			for _, secret := range secrets {
-				t.Run(fmt.Sprintf("secret %q still exists", secret.Name), func(t *testing.T) {
-					_, err := k8sh.Clientset.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
-					require.NoError(t, err)
-				})
-
-				// cleanup the secrets created for the test
-				t.Run(fmt.Sprintf("delete secret %q", secret.Name), func(t *testing.T) {
-					err := k8sh.Clientset.CoreV1().Secrets(secret.Namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{})
-					require.NoError(t, err)
-				})
-			}
+			// cleanup the secrets created for the test
+			t.Run(fmt.Sprintf("delete secret %q", secret.Name), func(t *testing.T) {
+				err := secretClient.Delete(ctx, secret.Name, metav1.DeleteOptions{})
+				require.NoError(t, err)
+			})
 		}
 
 		t.Run(fmt.Sprintf("no secrets in ns %q", ns.Name), func(t *testing.T) {
-			secrets, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).List(ctx, metav1.ListOptions{})
+			secrets, err := secretClient.List(ctx, metav1.ListOptions{})
 			require.NoError(t, err)
 
 			assert.Len(t, secrets.Items, 0)
 		})
-
-		t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
 	})
 }

tests/integration/object/user/opmask/opmask.go

@@ -20,24 +20,27 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/ceph/go-ceph/rgw/admin"
-	"github.com/coreos/pkg/capnslog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
-	utiladmin "github.com/rook/rook/tests/integration/object/util/admin"
+	"github.com/rook/rook/tests/integration/object/util/fixture"
+	"github.com/rook/rook/tests/integration/object/util/sharedstore"
+	"github.com/rook/rook/tests/integration/object/util/wait4"
 )
 
-func TestObjectStoreUserOpMask(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) {
+const Namespace = "test-useropmask"
+
+func TestObjectStoreUserOpMask(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
 	var (
-		defaultName = "test-useropmask"
+		defaultName = Namespace
+		objectStore = store.ObjectStore()
+		adminClient = store.AdminClient()
 
 		ns = &corev1.Namespace{
 			ObjectMeta: metav1.ObjectMeta{
@@ -55,47 +58,18 @@ func TestObjectStoreUserOpMask(t *testing.T, k8sh *utils.K8sHelper, installer *i
 				ClusterNamespace: objectStore.Namespace,
 			},
 		}
+
+		osuClient = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name)
 	)
 
-	t.Run("ObjectStoreUser keys", func(t *testing.T) {
-		if tlsEnable {
-			// Skip testing with and without TLS to reduce test time
-			t.Skip("skipping test for TLS enabled clusters")
-		}
+	t.Run("ObjectStoreUser op_mask", func(t *testing.T) {
+		ctx := t.Context()
 
-		var adminClient *admin.API
-		ctx := context.TODO()
-
-		t.Run(fmt.Sprintf("create ns %q", ns.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
+		fixture.RequireNamespace(t, k8sh, ns)
 
 		t.Run(fmt.Sprintf("create CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Create(ctx, &osu1, metav1.CreateOptions{})
-			require.NoError(t, err)
-
 			// user creation may be slow right after rgw start up
-			osuReady := utils.Retry(120, time.Second, "CephObjectStoreUser is Ready", func() bool {
-				liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				if liveOsu.Status == nil {
-					return false
-				}
-
-				return liveOsu.Status.Phase == string(cephv1.ConditionReady)
-			})
-			require.True(t, osuReady)
-		})
-
-		// wait for cosu user to be ready so we know rgw admin api is ready
-		t.Run("setup rgw admin api client", func(t *testing.T) {
-			var err error
-			adminClient, err = utiladmin.NewAdminClient(objectStore, installer, k8sh, tlsEnable)
-			require.NoError(t, err)
+			wait4.RequireCreate(ctx, t, osuClient, &osu1, wait4.ObjectStoreUser, wait4.TimeoutLong)
 		})
 
 		t.Run(fmt.Sprintf("verify op_mask set on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
@@ -105,95 +79,86 @@ func TestObjectStoreUserOpMask(t *testing.T, k8sh *utils.K8sHelper, installer *i
 		})
 
 		t.Run(fmt.Sprintf("update op-mask on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveOsu.Spec.OpMask = &[]cephv1.ObjectUserOpMask{"read"}
 
-			_, err = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Update(ctx, liveOsu, metav1.UpdateOptions{})
+			_, err = osuClient.Update(ctx, liveOsu, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("verify op_mask set on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			converged := utils.Retry(30, time.Second, "OpMask updated", func() bool {
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, `rgw user op_mask is "read"`, func(ctx context.Context) error {
 				liveUser, err := adminClient.GetUser(ctx, admin.User{ID: osu1.Name})
 				if err != nil {
-					return false
+					return err
 				}
 
-				return liveUser.OpMask == "read"
+				if liveUser.OpMask != "read" {
+					return fmt.Errorf("op_mask not yet %q: %q", "read", liveUser.OpMask)
+				}
+				return nil
 			})
-
-			assert.True(t, converged)
 		})
 
 		t.Run(fmt.Sprintf("remove op-mask on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveOsu.Spec.OpMask = nil
 
-			_, err = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Update(ctx, liveOsu, metav1.UpdateOptions{})
+			_, err = osuClient.Update(ctx, liveOsu, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("verify op_mask set on CephObjectStoreUser %q returns to default", osu1.Name), func(t *testing.T) {
-			converged := utils.Retry(30, time.Second, "OpMask updated", func() bool {
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, "rgw user op_mask returns to default", func(ctx context.Context) error {
 				liveUser, err := adminClient.GetUser(ctx, admin.User{ID: osu1.Name})
 				if err != nil {
-					return false
+					return err
 				}
 
-				return liveUser.OpMask == "read, write, delete"
+				if liveUser.OpMask != "read, write, delete" {
+					return fmt.Errorf("op_mask not yet default: %q", liveUser.OpMask)
+				}
+				return nil
 			})
-
-			assert.True(t, converged)
 		})
 
 		t.Run(fmt.Sprintf("set empty op-mask (remove all ops) on CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
+			liveOsu, err := osuClient.Get(ctx, osu1.Name, metav1.GetOptions{})
 			require.NoError(t, err)
 
 			liveOsu.Spec.OpMask = &[]cephv1.ObjectUserOpMask{}
 
-			_, err = k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Update(ctx, liveOsu, metav1.UpdateOptions{})
+			_, err = osuClient.Update(ctx, liveOsu, metav1.UpdateOptions{})
 			require.NoError(t, err)
 		})
 
 		t.Run(fmt.Sprintf("verify op_mask set on CephObjectStoreUser %q is empty", osu1.Name), func(t *testing.T) {
-			converged := utils.Retry(30, time.Second, "OpMask updated", func() bool {
+			wait4.AssertEventually(ctx, t, wait4.TimeoutShort, "rgw user op_mask is empty", func(ctx context.Context) error {
 				liveUser, err := adminClient.GetUser(ctx, admin.User{ID: osu1.Name})
 				if err != nil {
-					return false
+					return err
 				}
 
-				return liveUser.OpMask == "<none>"
+				if liveUser.OpMask != "<none>" {
+					return fmt.Errorf("op_mask not yet empty: %q", liveUser.OpMask)
+				}
+				return nil
 			})
-
-			assert.True(t, converged)
 		})
 
 		t.Run(fmt.Sprintf("delete CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Delete(ctx, osu1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "CephObjectStoreUser is absent", func() bool {
-				_, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
+			wait4.AssertDelete(ctx, t, osuClient, osu1.Name, wait4.TimeoutShort)
 		})
 
 		t.Run(fmt.Sprintf("no CephObjectStoreUsers in ns %q", ns.Name), func(t *testing.T) {
-			osus, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).List(ctx, metav1.ListOptions{})
+			osus, err := osuClient.List(ctx, metav1.ListOptions{})
 			require.NoError(t, err)
 
 			assert.Len(t, osus.Items, 0)
 		})
-
-		t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
 	})
 }

tests/integration/object/util/client/admin.go

@@ -14,11 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package admin
+// Package client builds the clients the object tests use to reach a
+// CephObjectStore — the rgw admin and SNS clients and the S3 credentials and
+// endpoint they need — along with the store's TLS certificate and a
+// verification-skipping HTTP client for the TLS pass.
+package client
 
 import (
 	"context"
-	"crypto/tls"
 	"net/http"
 
 	"github.com/ceph/go-ceph/rgw/admin"
@@ -27,29 +30,22 @@ import (
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
-	utils3 "github.com/rook/rook/tests/integration/object/util/s3"
 )
 
 func NewAdminClient(objectStore *cephv1.CephObjectStore, installer *installer.CephInstaller, k8sh *utils.K8sHelper, tlsEnable bool) (*admin.API, error) {
-	accessKey, secretKey, err := utils3.GetS3Credentials(objectStore, installer)
+	accessKey, secretKey, err := GetS3Credentials(objectStore, installer)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get s3 credentials")
 	}
 
-	endpoint, err := utils3.GetS3Endpoint(objectStore, k8sh, tlsEnable)
+	endpoint, err := GetS3Endpoint(objectStore, k8sh, tlsEnable)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get s3 endpoint")
 	}
 
 	httpClient := &http.Client{}
-
 	if tlsEnable {
-		httpClient.Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{
-				// nolint:gosec // skip TLS verification as this is a test
-				InsecureSkipVerify: true,
-			},
-		}
+		httpClient = InsecureHTTPClient()
 	}
 
 	adminClient, err := admin.New(endpoint, accessKey, secretKey, httpClient)

tests/integration/object/util/client/s3.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package s3
+package client
 
 import (
 	"context"
@@ -64,12 +64,12 @@ func GetS3Endpoint(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, t
 		return "", errors.Wrap(err, "failed to get objectstore svc")
 	}
 
-	schema := "http://"
+	schema, port := "http://", "80"
 	if tlsEnable {
-		schema = "https://"
+		schema, port = "https://", "443"
 	}
 
-	endpoint := schema + svc.Spec.ClusterIP + ":80"
+	endpoint := schema + svc.Spec.ClusterIP + ":" + port
 
 	return endpoint, nil
 }

tests/integration/object/util/client/sns.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package sns
+package client
 
 import (
 	"context"
@@ -29,7 +29,6 @@ import (
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
-	utils3 "github.com/rook/rook/tests/integration/object/util/s3"
 )
 
 // based on the s3 endpoint example in the official docs:
@@ -48,16 +47,15 @@ func (r *snsResolverV2) ResolveEndpoint(ctx context.Context, params sns.Endpoint
 	}, nil
 }
 
-func NewClient(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) {
+func NewSNSClient(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) {
 	ctx := context.TODO()
 
-	accessKey, secretKey, err := utils3.GetS3Credentials(objectStore, installer)
+	accessKey, secretKey, err := GetS3Credentials(objectStore, installer)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get s3 credentials")
 	}
 
-	cfg, err := config.LoadDefaultConfig(
-		ctx,
+	loadOpts := []func(*config.LoadOptions) error{
 		config.WithRegion("us-east-1"),
 		config.WithCredentialsProvider(
 			credentials.NewStaticCredentialsProvider(
@@ -66,12 +64,17 @@ func NewClient(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, insta
 				"",
 			),
 		),
-	)
+	}
+	if tlsEnable {
+		loadOpts = append(loadOpts, config.WithHTTPClient(InsecureHTTPClient()))
+	}
+
+	cfg, err := config.LoadDefaultConfig(ctx, loadOpts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load default aws config")
 	}
 
-	endpoint, err := utils3.GetS3Endpoint(objectStore, k8sh, tlsEnable)
+	endpoint, err := GetS3Endpoint(objectStore, k8sh, tlsEnable)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get sns endpoint")
 	}

tests/integration/object/util/client/tls.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2026 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"context"
+	cryptotls "crypto/tls"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/rook/rook/tests/framework/utils"
+)
+
+// InsecureHTTPClient returns an http.Client that skips TLS verification, for
+// reaching a test CephObjectStore served with a cluster-signed cert the test
+// process does not trust.
+func InsecureHTTPClient() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			// nolint:gosec // skip TLS verification as this is a test
+			TLSClientConfig: &cryptotls.Config{InsecureSkipVerify: true},
+		},
+	}
+}
+
+// GenerateRgwTLSCertSecret generates a cluster-signed TLS certificate for
+// rgwServiceName and stores it as an Opaque Secret named secretName in
+// namespace. The secret holds the concatenated key, certificate, and CA bundle
+// under the data key "cert", which is the format a CephObjectStore expects when
+// referenced via Gateway.SSLCertificateRef.
+func GenerateRgwTLSCertSecret(t *testing.T, k8sh *utils.K8sHelper, namespace, secretName, rgwServiceName string) {
+	ctx := context.TODO()
+
+	root, err := utils.FindRookRoot()
+	require.NoError(t, err, "failed to get rook root")
+
+	tlscertdir := t.TempDir()
+	cmdArgs := utils.CommandArgs{
+		Command: filepath.Join(root, "tests/scripts/generate-tls-config.sh"),
+		CmdArgs: []string{tlscertdir, rgwServiceName, namespace},
+	}
+	cmdOut := utils.ExecuteCommand(cmdArgs)
+	require.NoError(t, cmdOut.Err)
+
+	tlsKeyIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".key"))
+	require.NoError(t, err)
+	tlsCertIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".crt"))
+	require.NoError(t, err)
+	tlsCaCertIn, err := os.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".ca"))
+	require.NoError(t, err)
+
+	secretCertOut := fmt.Sprintf("%s%s%s", tlsKeyIn, tlsCertIn, tlsCaCertIn)
+	tlsK8sSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"cert": []byte(secretCertOut),
+		},
+	}
+	_, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(ctx, tlsK8sSecret, metav1.CreateOptions{})
+	require.NoError(t, err)
+}

tests/integration/object/util/fixture/fixture.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2026 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package fixture provides setup helpers for resources whose teardown is pure
+// cleanup rather than part of what a test asserts. Each helper creates the
+// resource and registers its deletion via t.Cleanup, so the resource is
+// removed even when the test fails partway through and never reaches its
+// trailing teardown steps — important because the tests share one cluster.
+//
+// Deletions that are themselves assertions (e.g. "deleting the OBC must not
+// delete the user") do not belong here; keep those as ordered subtests.
+package fixture
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/rook/rook/tests/framework/utils"
+)
+
+// The helpers manage their own contexts rather than taking one from the
+// caller: cleanup runs after the test finishes, when a test-scoped context
+// (e.g. t.Context()) has already been canceled.
+
+// RequireNamespace creates ns and registers its deletion via t.Cleanup.
+// Deletion is issued without waiting for it to complete; namespace teardown is
+// asynchronous and nothing in these tests depends on it finishing.
+func RequireNamespace(t *testing.T, k8sh *utils.K8sHelper, ns *corev1.Namespace) {
+	t.Helper()
+
+	_, err := k8sh.Clientset.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		err := k8sh.Clientset.CoreV1().Namespaces().Delete(context.Background(), ns.Name, metav1.DeleteOptions{})
+		if err != nil && !kerrors.IsNotFound(err) {
+			t.Errorf("failed to delete namespace %q: %v", ns.Name, err)
+		}
+	})
+}
+
+// RequireStorageClass creates sc and registers its deletion via t.Cleanup.
+func RequireStorageClass(t *testing.T, k8sh *utils.K8sHelper, sc *storagev1.StorageClass) {
+	t.Helper()
+
+	_, err := k8sh.Clientset.StorageV1().StorageClasses().Create(context.Background(), sc, metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		err := k8sh.Clientset.StorageV1().StorageClasses().Delete(context.Background(), sc.Name, metav1.DeleteOptions{})
+		if err != nil && !kerrors.IsNotFound(err) {
+			t.Errorf("failed to delete storageclass %q: %v", sc.Name, err)
+		}
+	})
+}

tests/integration/object/util/fixture/obc.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2026 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fixture
+
+import (
+	storagev1 "k8s.io/api/storage/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+)
+
+// StorageClass returns a StorageClass for the lib-bucket-provisioner bucket
+// provisioner backed by objectStore, for use by ObjectBucketClaims.
+func StorageClass(name string, objectStore *cephv1.CephObjectStore) *storagev1.StorageClass {
+	return &storagev1.StorageClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
+		Parameters: map[string]string{
+			"objectStoreName":      objectStore.Name,
+			"objectStoreNamespace": objectStore.Namespace,
+		},
+	}
+}

tests/integration/object/util/secrets/secrets.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2026 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package secrets holds verification helpers for the Secret references that
+// object CRDs publish in their status, shared by more than one object test
+// package. Verification helpers for a different resource belong in a package
+// named for that resource, not here.
+package secrets
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/tests/framework/utils"
+	"github.com/rook/rook/tests/integration/object/util/wait4"
+)
+
+// RequireStatusRefs runs a subtest named desc that waits for the status secret
+// references returned by refsOf to reach len(expected) — the controller
+// repopulates them asynchronously after a reconcile — and then asserts each ref
+// against its live secret via AssertRefs. refsOf must tolerate an unpopulated
+// status (return nil).
+func RequireStatusRefs[T, L runtime.Object](
+	t *testing.T,
+	k8sh *utils.K8sHelper,
+	client wait4.NamespacedWatcher[T, L],
+	name, desc string,
+	refsOf func(T) []cephv1.SecretReference,
+	expected ...*corev1.Secret,
+) {
+	t.Helper()
+	t.Run(desc, func(t *testing.T) {
+		ctx := t.Context()
+		live := wait4.RequireCondition(ctx, t, client, name, func(obj T) bool {
+			return len(refsOf(obj)) == len(expected)
+		}, wait4.TimeoutShort)
+		AssertRefs(ctx, t, k8sh, refsOf(live), expected...)
+	})
+}
+
+// AssertRefs verifies that refs contains exactly one SecretReference per
+// expected Secret and that each matches the live secret's Name, Namespace,
+// UID, and ResourceVersion. Live secrets are fetched from each expected
+// secret's own namespace.
+func AssertRefs(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, refs []cephv1.SecretReference, expected ...*corev1.Secret) {
+	t.Helper()
+
+	assert.Len(t, refs, len(expected))
+
+	for _, secret := range expected {
+		secretRef, err := findSecretRef(refs, secret.Name)
+		require.NoError(t, err)
+
+		// fetch the live secret for UID and ResourceVersion
+		liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
+		require.NoError(t, err)
+
+		assert.Equal(t, liveSecret.Name, secretRef.Name)
+		assert.Equal(t, liveSecret.Namespace, secretRef.Namespace)
+		assert.Equal(t, liveSecret.UID, secretRef.UID)
+		assert.Equal(t, liveSecret.ResourceVersion, secretRef.ResourceVersion)
+	}
+}
+
+// findSecretRef returns the SecretReference for the named secret.
+func findSecretRef(refs []cephv1.SecretReference, name string) (cephv1.SecretReference, error) {
+	for _, ref := range refs {
+		if ref.Name == name {
+			return ref, nil
+		}
+	}
+	return cephv1.SecretReference{}, fmt.Errorf("secretReference for secret %q not found", name)
+}

tests/integration/object/util/sharedstore/sharedstore.go

@@ -22,59 +22,154 @@ package sharedstore
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
-	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/tests/framework/utils"
+	"github.com/aws/aws-sdk-go-v2/service/sns"
+	"github.com/ceph/go-ceph/rgw/admin"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/tests/framework/installer"
+	"github.com/rook/rook/tests/framework/utils"
+	"github.com/rook/rook/tests/integration/object/util/client"
+	"github.com/rook/rook/tests/integration/object/util/wait4"
 )
 
+type Sharedstore struct {
+	adminClient *admin.API
+	snsClient   *sns.Client
+	objectStore *cephv1.CephObjectStore
+	tlsEnable   bool
+	destroy     func()
+}
+
+func (s *Sharedstore) AdminClient() *admin.API {
+	return s.adminClient
+}
+
+func (s *Sharedstore) SnsClient() *sns.Client {
+	return s.snsClient
+}
+
+func (s *Sharedstore) TLSEnabled() bool {
+	return s.tlsEnable
+}
+
+func (s *Sharedstore) ObjectStore() *cephv1.CephObjectStore {
+	return s.objectStore
+}
+
+func (s *Sharedstore) Destroy() {
+	s.destroy()
+}
+
 // Setup creates a shared CephObjectStore and its NodePort Service in
 // ObjectStoreNamespace, waits for the store to become Ready, and returns a
 // teardown function that deletes both resources. The teardown function should
 // be deferred by the caller.
-func Setup(t *testing.T, k8sh *utils.K8sHelper) (*cephv1.CephObjectStore, func()) {
+func Create(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool, allowedNamespaces ...string) *Sharedstore {
 	t.Helper()
+
+	s := &Sharedstore{tlsEnable: tlsEnable}
 	ctx := context.TODO()
+	ns := "object-ns"
+	storeName := "sharedstore"
+
+	// securePort is the in-container RGW TLS listener port. The NodePort Service
+	// below exposes the conventional 443 externally and forwards to it.
+	const securePort int32 = 8443
+	certSecretName := storeName + "-tls"
+	rgwServiceName := "rook-ceph-rgw-" + storeName
+
+	realm := &cephv1.CephObjectRealm{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      storeName,
+			Namespace: ns,
+		},
+		Spec: cephv1.ObjectRealmSpec{
+			DefaultRealm: true,
+		},
+	}
+
+	zoneGroup := &cephv1.CephObjectZoneGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      storeName,
+			Namespace: ns,
+		},
+		Spec: cephv1.ObjectZoneGroupSpec{
+			Realm: realm.Name,
+		},
+	}
+
+	zone := &cephv1.CephObjectZone{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      storeName,
+			Namespace: ns,
+		},
+		Spec: cephv1.ObjectZoneSpec{
+			ZoneGroup: zoneGroup.Name,
+			SharedPools: cephv1.ObjectSharedPoolsSpec{
+				PoolPlacements: []cephv1.PoolPlacementSpec{
+					{
+						Name:             "default",
+						Default:          true,
+						MetadataPoolName: storeName + ".rgw.buckets.index",
+						DataPoolName:     storeName + ".rgw.buckets.data",
+						StorageClasses: []cephv1.PlacementStorageClassSpec{
+							{
+								Name:         "FOO",
+								DataPoolName: storeName + ".rgw.buckets.data.foo",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	poolNames := map[string]string{
+		"rgw.root":                          ".rgw.root",
+		storeName + ".rgw.control":          "",
+		storeName + ".rgw.meta":             "",
+		storeName + ".rgw.log":              "",
+		storeName + ".rgw.otp":              "",
+		storeName + ".rgw.buckets.index":    "",
+		storeName + ".rgw.buckets.data":     "",
+		storeName + ".rgw.buckets.data.foo": "",
+	}
 
 	objectStore := &cephv1.CephObjectStore{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test-shared",
-			Namespace: "object-ns",
+			Name:      storeName,
+			Namespace: ns,
 		},
 		Spec: cephv1.ObjectStoreSpec{
-			MetadataPool: cephv1.PoolSpec{
-				Replicated: cephv1.ReplicatedSpec{
-					Size:                   1,
-					RequireSafeReplicaSize: false,
-				},
-			},
-			DataPool: cephv1.PoolSpec{
-				Replicated: cephv1.ReplicatedSpec{
-					Size:                   1,
-					RequireSafeReplicaSize: false,
-				},
+			Zone: cephv1.ZoneSpec{
+				Name: zone.Name,
 			},
 			Gateway: cephv1.GatewaySpec{
 				Port:      80,
 				Instances: 1,
 			},
-			// Allow CephObjectStoreUser resources from every namespace used by
-			// the sub-package tests under tests/integration/object.
-			AllowUsersInNamespaces: []string{
-				"test-bucketowner",
-				"test-topickafka",
-				"test-usercaps",
-				"test-userkeys",
-				"test-useropmask",
-			},
+			AllowUsersInNamespaces: allowedNamespaces,
 		},
 	}
 
+	if tlsEnable {
+		// TLS-only store: omit the plain Port and serve https on securePort,
+		// referencing the cert secret generated below.
+		objectStore.Spec.Gateway = cephv1.GatewaySpec{
+			SecurePort:        securePort,
+			SSLCertificateRef: certSecretName,
+			Instances:         1,
+		}
+	}
+
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			// The service name must match ObjectStoreName so that
@@ -101,30 +196,142 @@ func Setup(t *testing.T, k8sh *utils.K8sHelper) (*cephv1.CephObjectStore, func()
 		},
 	}
 
-	_, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{})
-	require.NoError(t, err)
-
-	osReady := utils.Retry(180, time.Second, "shared CephObjectStore is Ready", func() bool {
-		liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{})
-		if err != nil {
-			return false
-		}
-		if liveOs.Status == nil {
-			return false
-		}
-		return liveOs.Status.Phase == cephv1.ConditionReady
-	})
-	require.True(t, osReady, "shared CephObjectStore did not become Ready")
-
-	_, err = k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, svc, metav1.CreateOptions{})
-	require.NoError(t, err)
-
-	return objectStore, func() {
-		if err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}); err != nil {
-			t.Logf("failed to delete shared object store service: %v", err)
-		}
-		if err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}); err != nil {
-			t.Logf("failed to delete shared CephObjectStore: %v", err)
+	if tlsEnable {
+		// Expose the conventional https port externally, forwarding to the RGW
+		// TLS container port.
+		svc.Spec.Ports = []corev1.ServicePort{
+			{
+				Name:       "https",
+				Port:       443,
+				Protocol:   corev1.ProtocolTCP,
+				TargetPort: intstr.FromInt(int(securePort)),
+			},
 		}
 	}
+
+	canaryUser := &cephv1.CephObjectStoreUser{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      storeName + "-canary",
+			Namespace: ns,
+		},
+		Spec: cephv1.ObjectStoreUserSpec{
+			Store:            objectStore.Name,
+			ClusterNamespace: objectStore.Namespace,
+		},
+	}
+
+	var pools []*cephv1.CephBlockPool
+
+	for k, v := range poolNames {
+		// .rgw.root is the cluster-global realm metadata pool shared by every
+		// object store. The operator sets pg_num_min=8 on it when reconciling a
+		// normal CephObjectStore (e.g. the COSI test store), so it must have
+		// pg_num>=8 or that reconcile fails with "pg_num_min 8 > pg_num 1". The
+		// store-private pools are not shared and can stay at the minimal pg_num.
+		pgNum := "1"
+		if v == ".rgw.root" {
+			pgNum = "8"
+		}
+		pools = append(pools, &cephv1.CephBlockPool{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      k,
+				Namespace: ns,
+			},
+			Spec: cephv1.NamedBlockPoolSpec{
+				Name: v,
+				PoolSpec: cephv1.PoolSpec{
+					Replicated: cephv1.ReplicatedSpec{
+						Size:                   1,
+						RequireSafeReplicaSize: false,
+					},
+					Parameters: map[string]string{
+						"pg_autoscale_mode": "off",
+						"pg_num":            pgNum,
+					},
+				},
+			},
+		})
+	}
+
+	ceph := k8sh.RookClientset.CephV1()
+
+	{
+		_, err := ceph.CephObjectRealms(ns).Create(ctx, realm, metav1.CreateOptions{})
+		require.NoError(t, err)
+	}
+
+	{
+		_, err := ceph.CephObjectZoneGroups(ns).Create(ctx, zoneGroup, metav1.CreateOptions{})
+		require.NoError(t, err)
+	}
+
+	{
+		_, err := ceph.CephObjectZones(ns).Create(ctx, zone, metav1.CreateOptions{})
+		require.NoError(t, err)
+	}
+
+	for _, p := range pools {
+		_, err := ceph.CephBlockPools(ns).Create(ctx, p, metav1.CreateOptions{})
+		require.NoError(t, err)
+	}
+
+	// the cert secret must exist before the store is created so the operator can
+	// mount it when the rgw pod first reconciles
+	if tlsEnable {
+		client.GenerateRgwTLSCertSecret(t, k8sh, ns, certSecretName, rgwServiceName)
+	}
+
+	wait4.RequireCreate(ctx, t, ceph.CephObjectStores(ns), objectStore, wait4.ObjectStore,
+		3*time.Minute, "shared CephObjectStore did not become Ready")
+
+	{
+		_, err := k8sh.Clientset.CoreV1().Services(ns).Create(ctx, svc, metav1.CreateOptions{})
+		require.NoError(t, err)
+	}
+
+	t.Run("setup rgw admin api client", func(t *testing.T) {
+		// user creation may be slow right after rgw start up
+		wait4.RequireCreate(ctx, t, ceph.CephObjectStoreUsers(ns), canaryUser, wait4.ObjectStoreUser, 2*time.Minute)
+
+		// cleanup the canary user immediately so it does not block the
+		// CephObjectStore's deletion later
+		wait4.AssertDelete(ctx, t, ceph.CephObjectStoreUsers(ns), canaryUser.Name, time.Minute)
+
+		// the canary user becoming ready tells us the rgw admin api is ready
+		var err error
+		s.adminClient, err = client.NewAdminClient(objectStore, installer, k8sh, tlsEnable)
+		require.NoError(t, err)
+
+		s.snsClient, err = client.NewSNSClient(objectStore, k8sh, installer, tlsEnable)
+		require.NoError(t, err)
+	})
+
+	s.objectStore = objectStore
+
+	s.destroy = func() {
+		t.Run(fmt.Sprintf("destroy CephObjectStore %s", objectStore.Name), func(t *testing.T) {
+			wait4.AssertDelete(ctx, t, k8sh.Clientset.CoreV1().Services(ns), svc.Name, 30*time.Second)
+			wait4.AssertDelete(ctx, t, ceph.CephObjectStores(ns), objectStore.Name, 5*time.Minute)
+
+			// The multisite CRs must be deleted before the pools. Their deletion
+			// finalizers run radosgw-admin, which reads the realm/zonegroup/zone
+			// metadata stored in the .rgw.root pool. Deleting the pools first
+			// removes that metadata, causing radosgw-admin to fail (exit status 2)
+			// and the zone controller to loop forever, so the CephObjectZone CR
+			// never finishes deleting.
+			wait4.AssertDelete(ctx, t, ceph.CephObjectZones(ns), zone.Name, 2*time.Minute)
+			wait4.AssertDelete(ctx, t, ceph.CephObjectZoneGroups(ns), zoneGroup.Name, time.Minute)
+			wait4.AssertDelete(ctx, t, ceph.CephObjectRealms(ns), realm.Name, time.Minute)
+
+			for _, p := range pools {
+				wait4.AssertDelete(ctx, t, ceph.CephBlockPools(ns), p.Name, time.Minute)
+			}
+
+			if tlsEnable {
+				wait4.AssertDelete(ctx, t, k8sh.Clientset.CoreV1().Secrets(ns), certSecretName, 30*time.Second)
+			}
+		})
+	}
+
+	return s
 }

tests/integration/object/util/wait4/ready.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2026 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package wait4
+
+import (
+	bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+)
+
+// ObjectStore reports whether a CephObjectStore has been reconciled to Ready.
+func ObjectStore(os *cephv1.CephObjectStore) bool {
+	return os.Status != nil && os.Status.Phase == cephv1.ConditionReady
+}
+
+// ObjectStoreUser reports whether a CephObjectStoreUser has been reconciled to
+// Ready.
+func ObjectStoreUser(u *cephv1.CephObjectStoreUser) bool {
+	return ObjectStoreUserPhase(string(cephv1.ConditionReady))(u)
+}
+
+// ObjectStoreUserPhase returns a predicate reporting whether a
+// CephObjectStoreUser's .status.phase matches phase. The parameter is a plain
+// string because the cephv1 phase constants are a mix of ConditionType and
+// ConditionReason (e.g. string(cephv1.ReconcileFailed)).
+func ObjectStoreUserPhase(phase string) func(*cephv1.CephObjectStoreUser) bool {
+	return func(u *cephv1.CephObjectStoreUser) bool {
+		return u.Status != nil && u.Status.Phase == phase
+	}
+}
+
+// BucketTopic reports whether a CephBucketTopic has been reconciled to Ready
+// with its ARN populated. Note that .status.ARN is nil while .status.phase ==
+// Reconciling, so the phase check alone is not sufficient.
+func BucketTopic(bt *cephv1.CephBucketTopic) bool {
+	return BucketTopicPhase(string(cephv1.ConditionReady))(bt) && bt.Status.ARN != nil
+}
+
+// BucketTopicPhase returns a predicate reporting whether a CephBucketTopic's
+// .status.phase matches phase. The parameter is a plain string because the
+// cephv1 phase constants are a mix of ConditionType and ConditionReason (e.g.
+// string(cephv1.ReconcileFailed)).
+func BucketTopicPhase(phase string) func(*cephv1.CephBucketTopic) bool {
+	return func(bt *cephv1.CephBucketTopic) bool {
+		return bt.Status != nil && bt.Status.Phase == phase
+	}
+}
+
+// OBCBound reports whether an ObjectBucketClaim has reached the Bound phase.
+func OBCBound(obc *bktv1alpha1.ObjectBucketClaim) bool {
+	return obc.Status.Phase == bktv1alpha1.ObjectBucketClaimStatusPhaseBound
+}
+
+// OBBound reports whether an ObjectBucket has reached the Bound phase.
+func OBBound(ob *bktv1alpha1.ObjectBucket) bool {
+	return ob.Status.Phase == bktv1alpha1.ObjectBucketStatusPhaseBound
+}

tests/integration/object/util/wait4/wait4.go

@@ -0,0 +1,488 @@
+/*
+Copyright 2026 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package wait4
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+
+	"github.com/rook/rook/tests/framework/utils"
+)
+
+// Default timeouts for waiting on operator reconciliation in the object
+// tests. Bespoke waits (e.g. the shared store's multi-minute teardown ladder)
+// still pass explicit durations.
+const (
+	// TimeoutShort covers routine status changes on an already-running
+	// operator and rgw: condition waits, deletions, field updates.
+	TimeoutShort = 40 * time.Second
+	// TimeoutMedium covers resource creation that needs a full reconcile,
+	// such as a CephBucketTopic becoming Ready with its ARN set.
+	TimeoutMedium = time.Minute
+	// TimeoutLong covers first-reconcile waits that may race rgw startup,
+	// such as CephObjectStoreUser creation right after the store comes up.
+	TimeoutLong = 2 * time.Minute
+)
+
+// NamespacedWatcher is satisfied by any typed namespaced client (corev1
+// ServiceInterface, cephv1 CephObjectStoreUserInterface, etc.). T is the
+// resource's pointer type and L its list type, both inferred at the call site
+// from the client's Get/List return types.
+type NamespacedWatcher[T, L runtime.Object] interface {
+	Get(ctx context.Context, name string, opts metav1.GetOptions) (T, error)
+	List(ctx context.Context, opts metav1.ListOptions) (L, error)
+	Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
+}
+
+// nameScopedListWatch returns a ListWatch restricted to a single resource name.
+func nameScopedListWatch[T, L runtime.Object](ctx context.Context, client NamespacedWatcher[T, L], name string) *cache.ListWatch {
+	fs := fields.OneTermEqualSelector("metadata.name", name).String()
+	return &cache.ListWatch{
+		ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+			opts.FieldSelector = fs
+			return client.List(ctx, opts)
+		},
+		WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+			opts.FieldSelector = fs
+			return client.Watch(ctx, opts)
+		},
+	}
+}
+
+// heartbeatInterval is how often a watch-based wait logs progress while it is
+// still blocked.
+const heartbeatInterval = 15 * time.Second
+
+// heartbeat logs a periodic progress line for a watch-based wait that would
+// otherwise block silently, so a slow reconcile shows up in CI output instead
+// of looking like a hang. It logs nothing for waits shorter than
+// heartbeatInterval; the caller must invoke the returned stop func when the
+// wait returns.
+func heartbeat(t *testing.T, desc string) (stop func()) {
+	t.Helper()
+	done := make(chan struct{})
+	go func() {
+		ticker := time.NewTicker(heartbeatInterval)
+		defer ticker.Stop()
+		for elapsed := heartbeatInterval; ; elapsed += heartbeatInterval {
+			select {
+			case <-done:
+				return
+			case <-ticker.C:
+				t.Logf("still waiting for %s (%s)", desc, elapsed)
+			}
+		}
+	}()
+	return func() { close(done) }
+}
+
+// waitForAbsent blocks until the named resource is gone or the timeout elapses.
+//
+// The wait is race-free against fast deletions: UntilWithSync performs an
+// initial List (scoped to the resource name via a field selector) before
+// starting the watch, and the precondition treats an empty store as "already
+// gone". Subsequent deletion events are guaranteed to be delivered because the
+// watch resumes from the List's resourceVersion.
+func waitForAbsent[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedWatcher[T, L], name string, timeout time.Duration) error {
+	var objType T
+	lw := nameScopedListWatch(ctx, client, name)
+
+	waitCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	precondition := func(store cache.Store) (bool, error) {
+		// lw is name-scoped, so an empty store unambiguously means the object
+		// was already gone by the time the reflector's initial list completed.
+		return len(store.List()) == 0, nil
+	}
+	condition := func(ev watch.Event) (bool, error) {
+		return ev.Type == watch.Deleted, nil
+	}
+
+	defer heartbeat(t, fmt.Sprintf("%T %q to be removed", objType, name))()
+
+	if _, err := watchtools.UntilWithSync(waitCtx, lw, objType, precondition, condition); err != nil {
+		return fmt.Errorf("%T %q is still present: %w", objType, name, err)
+	}
+	return nil
+}
+
+// waitForCondition blocks until the named resource satisfies cond or the timeout
+// elapses, returning the matching object. The initial List makes it race-free in
+// the same way as waitForAbsent: a resource that already satisfies cond is
+// detected immediately, and subsequent updates arrive as watch events.
+func waitForCondition[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedWatcher[T, L], name string, cond func(T) bool, timeout time.Duration) (T, error) {
+	var objType T
+	var match T
+	lw := nameScopedListWatch(ctx, client, name)
+
+	waitCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	precondition := func(store cache.Store) (bool, error) {
+		for _, o := range store.List() {
+			if typed, ok := o.(T); ok && cond(typed) {
+				match = typed
+				return true, nil
+			}
+		}
+		return false, nil
+	}
+	condition := func(ev watch.Event) (bool, error) {
+		switch ev.Type {
+		case watch.Added, watch.Modified:
+			if typed, ok := ev.Object.(T); ok && cond(typed) {
+				match = typed
+				return true, nil
+			}
+		case watch.Deleted:
+			return false, fmt.Errorf("%T %q was deleted while waiting for condition", objType, name)
+		}
+		return false, nil
+	}
+
+	defer heartbeat(t, fmt.Sprintf("%T %q to be ready", objType, name))()
+
+	if _, err := watchtools.UntilWithSync(waitCtx, lw, objType, precondition, condition); err != nil {
+		return match, fmt.Errorf("%T %q did not satisfy condition: %w", objType, name, err)
+	}
+	return match, nil
+}
+
+// NamespacedDeleter is a NamespacedWatcher that can also delete by name.
+type NamespacedDeleter[T, L runtime.Object] interface {
+	NamespacedWatcher[T, L]
+	Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
+}
+
+func deleteAndWait[T, L runtime.Object](ctx context.Context, t *testing.T, client NamespacedDeleter[T, L], name string, timeout time.Duration) error {
+	var objType T
+	if err := client.Delete(ctx, name, metav1.DeleteOptions{}); err != nil {
+		return fmt.Errorf("failed to delete %T %q: %w", objType, name, err)
+	}
+	return waitForAbsent[T, L](ctx, t, client, name, timeout)
+}
+
+// AssertDelete deletes a single named resource and blocks until it is gone,
+// reporting failures (including timeout) via assert.Fail (non-fatal). It returns
+// true on success. Prefer this for teardown/cleanup, so a stuck finalizer on one
+// resource does not prevent cleanup of the rest; use RequireDelete when later
+// steps depend on the deletion having completed.
+func AssertDelete[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedDeleter[T, L],
+	name string,
+	timeout time.Duration,
+	msgAndArgs ...interface{},
+) bool {
+	t.Helper()
+	if err := deleteAndWait(ctx, t, client, name, timeout); err != nil {
+		return assert.Fail(t, err.Error(), msgAndArgs...)
+	}
+	return true
+}
+
+// RequireDelete is the fatal counterpart of AssertDelete: it stops the test
+// (FailNow) if the resource cannot be deleted or is not gone before the timeout.
+func RequireDelete[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedDeleter[T, L],
+	name string,
+	timeout time.Duration,
+	msgAndArgs ...interface{},
+) {
+	t.Helper()
+	require.NoError(t, deleteAndWait(ctx, t, client, name, timeout), msgAndArgs...)
+}
+
+// AssertAbsent waits for a named resource to disappear without deleting it
+// itself, reporting failures via assert.Fail (non-fatal). Use it for resources
+// removed as a side effect of another action — e.g. an ObjectBucket that is
+// garbage-collected when its ObjectBucketClaim is deleted. To delete and wait,
+// use AssertDelete/RequireDelete.
+func AssertAbsent[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedWatcher[T, L],
+	name string,
+	timeout time.Duration,
+	msgAndArgs ...interface{},
+) bool {
+	t.Helper()
+	if err := waitForAbsent(ctx, t, client, name, timeout); err != nil {
+		return assert.Fail(t, err.Error(), msgAndArgs...)
+	}
+	return true
+}
+
+// RequireAbsent is the fatal counterpart of AssertAbsent.
+func RequireAbsent[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedWatcher[T, L],
+	name string,
+	timeout time.Duration,
+	msgAndArgs ...interface{},
+) {
+	t.Helper()
+	require.NoError(t, waitForAbsent(ctx, t, client, name, timeout), msgAndArgs...)
+}
+
+// NamespacedReadyWaiter is a NamespacedWatcher that can also create.
+type NamespacedReadyWaiter[T, L runtime.Object] interface {
+	NamespacedWatcher[T, L]
+	Create(ctx context.Context, obj T, opts metav1.CreateOptions) (T, error)
+}
+
+// createAndWaitReady creates obj and, when ready is non-nil, blocks until ready
+// reports true for the live object or the timeout elapses, returning the live
+// object. Pass a nil ready predicate for resources that are usable as soon as
+// they are created (Namespaces, Services, Secrets, ...).
+//
+// Unlike deletion, "ready" has no universal representation across resource
+// types, so the readiness check is supplied by the caller.
+func createAndWaitReady[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedReadyWaiter[T, L],
+	obj T,
+	ready func(T) bool,
+	timeout time.Duration,
+) (T, error) {
+	var zero T
+
+	created, err := client.Create(ctx, obj, metav1.CreateOptions{})
+	if err != nil {
+		return zero, fmt.Errorf("failed to create %T: %w", obj, err)
+	}
+	if ready == nil {
+		return created, nil
+	}
+
+	accessor, err := meta.Accessor(created)
+	if err != nil {
+		return zero, fmt.Errorf("failed to read metadata of %T: %w", created, err)
+	}
+
+	return waitForCondition[T, L](ctx, t, client, accessor.GetName(), ready, timeout)
+}
+
+// AssertCreate creates obj and waits for the ready predicate, reporting failures
+// (including timeout) via assert.Fail (non-fatal) like AssertDelete. It returns
+// the live object and true on success. Prefer this for cleanup/best-effort
+// creation where the test should continue on failure; for setup preconditions
+// use RequireCreate.
+func AssertCreate[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedReadyWaiter[T, L],
+	obj T,
+	ready func(T) bool,
+	timeout time.Duration,
+	msgAndArgs ...interface{},
+) (T, bool) {
+	t.Helper()
+	live, err := createAndWaitReady(ctx, t, client, obj, ready, timeout)
+	if err != nil {
+		var zero T
+		assert.Fail(t, err.Error(), msgAndArgs...)
+		return zero, false
+	}
+	return live, true
+}
+
+// RequireCreate is the fatal counterpart of AssertCreate: it stops the test
+// (FailNow) if the resource cannot be created or does not become ready, and
+// returns the live object. This is the common case for setup preconditions, so
+// the call site needs no follow-up require.True.
+func RequireCreate[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedReadyWaiter[T, L],
+	obj T,
+	ready func(T) bool,
+	timeout time.Duration,
+	msgAndArgs ...interface{},
+) T {
+	t.Helper()
+	live, err := createAndWaitReady(ctx, t, client, obj, ready, timeout)
+	require.NoError(t, err, msgAndArgs...)
+	return live
+}
+
+// AssertCondition waits for the named resource to satisfy cond, reporting
+// failures (including timeout) via assert.Fail (non-fatal). It returns the
+// matching object and true on success. Use it to wait on an already-created
+// resource — e.g. an OBC reaching Bound, or a status field changing after an
+// update; for the create-and-wait case use AssertCreate/RequireCreate.
+func AssertCondition[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedWatcher[T, L],
+	name string,
+	cond func(T) bool,
+	timeout time.Duration,
+	msgAndArgs ...interface{},
+) (T, bool) {
+	t.Helper()
+	match, err := waitForCondition(ctx, t, client, name, cond, timeout)
+	if err != nil {
+		var zero T
+		assert.Fail(t, err.Error(), msgAndArgs...)
+		return zero, false
+	}
+	return match, true
+}
+
+// RequireCondition is the fatal counterpart of AssertCondition.
+func RequireCondition[T, L runtime.Object](
+	ctx context.Context,
+	t *testing.T,
+	client NamespacedWatcher[T, L],
+	name string,
+	cond func(T) bool,
+	timeout time.Duration,
+	msgAndArgs ...interface{},
+) T {
+	t.Helper()
+	match, err := waitForCondition(ctx, t, client, name, cond, timeout)
+	require.NoError(t, err, msgAndArgs...)
+	return match
+}
+
+// AssertEventually polls cond until it returns nil or the timeout elapses,
+// reporting failure via assert.Fail (non-fatal). It is for state the
+// Kubernetes API cannot watch (the rgw admin API, S3, SNS); k8s resources
+// should use the watch-based helpers above. The poll loop is utils.Eventually;
+// this adds the assert-flavored, non-fatal reporting that pairs with the
+// AssertCreate/AssertDelete/AssertCondition family.
+//
+// cond must not call assert or require: report transient failures by returning
+// an error (which is surfaced in the timeout message), and make assertions on
+// captured state after the wait. cond receives a context bounded by the wait
+// deadline; thread it into the reads it performs.
+func AssertEventually(ctx context.Context, t *testing.T, timeout time.Duration, desc string, cond func(context.Context) error) bool {
+	t.Helper()
+	if err := utils.Eventually(ctx, t, timeout, desc, cond); err != nil {
+		return assert.Fail(t, err.Error())
+	}
+	return true
+}
+
+// RequireEventually is the fatal counterpart of AssertEventually: it stops
+// the test (FailNow) if cond does not succeed before the timeout. Use it
+// when subsequent statements consume the polled result.
+func RequireEventually(ctx context.Context, t *testing.T, timeout time.Duration, desc string, cond func(context.Context) error) {
+	t.Helper()
+	require.NoError(t, utils.Eventually(ctx, t, timeout, desc, cond))
+}
+
+// matchPodLog follows the logs of the first pod matching selector until match
+// returns true for a line, the stream ends, or the wait deadline passes.
+// Reaching the end of the stream without a match is an error: following the
+// stream (rather than scanning a snapshot) is what lets the wait distinguish
+// "absent" from "not logged yet".
+func matchPodLog(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, selector labels.Selector, timeout time.Duration, match func(line string) bool) error {
+	t.Helper()
+
+	waitCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(waitCtx, metav1.ListOptions{LabelSelector: selector.String()})
+	if err != nil {
+		return fmt.Errorf("failed to list pods: %w", err)
+	}
+	if len(pods.Items) == 0 {
+		return fmt.Errorf("no pods found with labels %v in namespace %q", selector, namespace)
+	}
+
+	// if there are multiple pods, just pick the first one
+	pod := pods.Items[0]
+	t.Logf("matching against logs of pod %q (first match for labels %v)", pod.Name, selector)
+
+	req := k8sh.Clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Follow: true})
+	logStream, err := req.Stream(waitCtx)
+	if err != nil {
+		return fmt.Errorf("failed to stream logs from pod %q: %w", pod.Name, err)
+	}
+	defer logStream.Close()
+
+	scanner := bufio.NewScanner(logStream)
+	for scanner.Scan() {
+		if match(scanner.Text()) {
+			return nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("no matching log line from pod %q: %w", pod.Name, err)
+	}
+	return fmt.Errorf("log stream from pod %q ended without a matching line", pod.Name)
+}
+
+// AssertPodLog follows the logs of the first pod matching selector and reports
+// failure via assert.Fail (non-fatal) if no line satisfies match before the
+// timeout. It is for log-based state the Kubernetes API cannot watch (e.g. an
+// operator logging a reconcile error); for a setup precondition use
+// RequirePodLog.
+func AssertPodLog(
+	ctx context.Context,
+	t *testing.T,
+	k8sh *utils.K8sHelper,
+	namespace string,
+	selector labels.Selector,
+	timeout time.Duration,
+	match func(line string) bool,
+	msgAndArgs ...interface{},
+) bool {
+	t.Helper()
+	if err := matchPodLog(ctx, t, k8sh, namespace, selector, timeout, match); err != nil {
+		return assert.Fail(t, err.Error(), msgAndArgs...)
+	}
+	return true
+}
+
+// RequirePodLog is the fatal counterpart of AssertPodLog.
+func RequirePodLog(
+	ctx context.Context,
+	t *testing.T,
+	k8sh *utils.K8sHelper,
+	namespace string,
+	selector labels.Selector,
+	timeout time.Duration,
+	match func(line string) bool,
+	msgAndArgs ...interface{},
+) {
+	t.Helper()
+	require.NoError(t, matchPodLog(ctx, t, k8sh, namespace, selector, timeout, match), msgAndArgs...)
+}