diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go index 46410b10f..07a1b4f5e 100644 --- a/tests/integration/ceph_object_test.go +++ b/tests/integration/ceph_object_test.go @@ -19,18 +19,11 @@ package integration import ( "context" "encoding/json" - "errors" "fmt" "strings" "testing" "time" - "github.com/aws/aws-sdk-go-v2/service/s3" - s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - smithy "github.com/aws/smithy-go" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -44,6 +37,7 @@ import ( "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" + bucketlifecycle "github.com/rook/rook/tests/integration/object/bucket/lifecycle" bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner" bucketpolicy "github.com/rook/rook/tests/integration/object/bucket/policy" bucketquota "github.com/rook/rook/tests/integration/object/bucket/quota" @@ -64,20 +58,6 @@ const ( var objectStoreServicePrefix = "rook-ceph-rgw-" -var lifecycleCmpOpts = cmp.Options{ - cmpopts.IgnoreUnexported( - s3types.LifecycleRule{}, - s3types.LifecycleExpiration{}, - s3types.LifecycleRuleFilter{}, - s3types.LifecycleRuleAndOperator{}, - s3types.AbortIncompleteMultipartUpload{}, - s3types.NoncurrentVersionExpiration{}, - s3types.NoncurrentVersionTransition{}, - s3types.Transition{}, - s3types.Tag{}, - ), -} - func TestCephObjectSuite(t *testing.T) { s := new(ObjectSuite) defer func(s *ObjectSuite) { @@ -215,6 +195,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install testObjectStoreOperations(s, helper, k8sh, settings, storeName, swiftAndKeystone) sharedObjectStore := sharedstore.Create(s.T(), k8sh, installer, tlsEnable, settings.Namespace, "sharedstore", 1, + bucketlifecycle.Namespace, bucketowner.Namespace, bucketpolicy.Namespace, bucketquota.Namespace, @@ -228,6 +209,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install ) defer sharedObjectStore.Destroy() + bucketlifecycle.TestObjectBucketClaimLifecycle(s.T(), k8sh, sharedObjectStore) bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, sharedObjectStore) bucketpolicy.TestObjectBucketClaimPolicy(s.T(), k8sh, sharedObjectStore) bucketquota.TestObjectBucketClaimQuota(s.T(), k8sh, sharedObjectStore) @@ -296,261 +278,6 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh logger.Info("OBC, Secret and ConfigMap created") }) - t.Run("OBC bucket lifecycle management", func(t *testing.T) { - bucketName := "bucket-lifecycle-test" - var obName string - var s3client *rgw.S3Agent - bucketLifecycle1 := ` -{ - "Rules":[ - { - "ID": "AbortIncompleteMultipartUploads", - "Status": "Enabled", - "Prefix": "", - "AbortIncompleteMultipartUpload": { - "DaysAfterInitiation": 1 - } - } - ] -} - ` - - // rules must be sorted by ID to be idempotent - bucketLifecycle2 := ` -{ - "Rules": [ - { - "ID": "AbortIncompleteMultipartUploads", - "Status": "Enabled", - "Prefix": "", - "AbortIncompleteMultipartUpload": { - "DaysAfterInitiation": 1 - } - }, - { - "ID": "ExpireAfter30Days", - "Status": "Enabled", - "Prefix": "", - "Expiration": { - "Days": 30 - } - } - ] -} - ` - - t.Run("create obc with bucketLifecycle", func(t *testing.T) { - newObc := v1alpha1.ObjectBucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: bucketName, - Namespace: namespace, - }, - Spec: v1alpha1.ObjectBucketClaimSpec{ - BucketName: bucketName, - StorageClassName: bucketStorageClassName, - AdditionalConfig: map[string]string{ - "bucketLifecycle": bucketLifecycle1, - }, - }, - } - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Create(ctx, &newObc, metav1.CreateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that bucketLifecycle is set - assert.Equal(t, bucketLifecycle1, obc.Spec.AdditionalConfig["bucketLifecycle"]) - - // as the tests are running external to k8s, the internal svc can't be used - labelSelector := "rgw=" + storeName - services, err := k8sh.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) - assert.Nil(t, err) - assert.Equal(t, 1, len(services.Items)) - s3endpoint := services.Items[0].Spec.ClusterIP + ":80" - - secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"]) - s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"]) - - insecure := objectStore.Spec.IsTLSEnabled() - s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil) - assert.Nil(t, err) - logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) - }) - - t.Run("lifecycle was applied verbatim to bucket", func(t *testing.T) { - liveLc, err := s3client.Client.GetBucketLifecycleConfiguration(ctx, &s3.GetBucketLifecycleConfigurationInput{ - Bucket: &bucketName, - }) - require.NoError(t, err) - - confLc := &s3types.BucketLifecycleConfiguration{} - err = json.Unmarshal([]byte(bucketLifecycle1), confLc) - require.NoError(t, err) - - assert.True(t, cmp.Equal(confLc.Rules, liveLc.Rules, lifecycleCmpOpts...), cmp.Diff(confLc.Rules, liveLc.Rules, lifecycleCmpOpts...)) - }) - - t.Run("update obc bucketLifecycle", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - - obc.Spec.AdditionalConfig["bucketLifecycle"] = bucketLifecycle2 - - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that updated bucketLifecycle is set - assert.Equal(t, bucketLifecycle2, obc.Spec.AdditionalConfig["bucketLifecycle"]) - }) - - t.Run("lifecycle update applied verbatim to bucket", func(t *testing.T) { - var liveLc *s3.GetBucketLifecycleConfigurationOutput - - confLc := &s3types.BucketLifecycleConfiguration{} - err = json.Unmarshal([]byte(bucketLifecycle2), confLc) - require.NoError(t, err) - - utils.Retry(20, time.Second, "lifecycle changed", func() bool { - liveLc, err = s3client.Client.GetBucketLifecycleConfiguration(ctx, &s3.GetBucketLifecycleConfigurationInput{ - Bucket: &bucketName, - }) - if err != nil { - return false - } - - return cmp.Equal(confLc.Rules, liveLc.Rules, lifecycleCmpOpts...) - }) - assert.True(t, cmp.Equal(confLc.Rules, liveLc.Rules, lifecycleCmpOpts...), cmp.Diff(confLc.Rules, liveLc.Rules, lifecycleCmpOpts...)) - }) - - t.Run("remove obc bucketLifecycle", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - - obc.Spec.AdditionalConfig = map[string]string{} - - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that bucketLifecycle is unset - assert.NotContains(t, obc.Spec.AdditionalConfig, "bucketLifecycle") - }) - - t.Run("lifecycle was removed from bucket", func(t *testing.T) { - cephcluster, err := k8sh.RookClientset.CephV1().CephClusters(namespace).Get(ctx, settings.ClusterName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("failed to get CephCluster: %v", err) - } - logger.Infof("CephCluster version: %s", cephcluster.Status.CephVersion.Version) - if strings.HasPrefix(cephcluster.Status.CephVersion.Version, "19.2.3") { - t.Skip("Waiting for rgw fix from regression in v19.2.3") - } - utils.Retry(20, time.Second, "lifecycle is gone", func() bool { - _, err = s3client.Client.GetBucketLifecycleConfiguration(ctx, &s3.GetBucketLifecycleConfigurationInput{ - Bucket: &bucketName, - }) - var apiErr smithy.APIError - if errors.As(err, &apiErr) { - return apiErr.ErrorCode() == "NoSuchLifecycleConfiguration" - } - return false - }) - require.Error(t, err) - var apiErr smithy.APIError - require.True(t, errors.As(err, &apiErr)) - assert.Equal(t, "NoSuchLifecycleConfiguration", apiErr.ErrorCode()) - }) - - t.Run("delete bucket", func(t *testing.T) { - err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Delete(ctx, bucketName, metav1.DeleteOptions{}) - assert.Nil(t, err) - - absent := utils.Retry(20, time.Second, "OBC is absent", func() bool { - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - return err != nil - }) - assert.True(t, absent) - - absent = utils.Retry(20, time.Second, "OB is absent", func() bool { - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - return err != nil - }) - assert.True(t, absent) - }) - }) - t.Run("delete CephObjectStore should be blocked by OBC bucket and CephObjectStoreUser", func(t *testing.T) { deleteObjectStore(t, k8sh, namespace, storeName) diff --git a/tests/integration/object/README.md b/tests/integration/object/README.md index 907a033b1..2b3bc175e 100644 --- a/tests/integration/object/README.md +++ b/tests/integration/object/README.md @@ -32,6 +32,7 @@ must not collide with std-lib package names (no `io`, no `http`). | test package | operator package | covers | |---|---|---| +| `bucket/lifecycle` | `object/bucket` | OBC bucketLifecycle management | | `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling | | `bucket/policy` | `object/bucket` | OBC bucketPolicy management | | `bucket/quota` | `object/bucket` | OBC maxObjects user quota + bucketMaxObjects/bucketMaxSize bucket quota | @@ -42,7 +43,7 @@ must not collide with std-lib package names (no `io`, no `http`). | `user/caps` | `object/user` | user capabilities | | `user/keys` | `object/user` | explicit S3 key management | | `user/opmask` | `object/user` | user op_mask | -| reserved: `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | +| reserved: `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | Shared utilities live under `util/`: @@ -208,7 +209,6 @@ verification; wire the dispatcher; delete the old code; retire | old test | target package(s) | still needs (build in that PR) | |---|---|---| -| `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | move `lifecycleCmpOpts` out of the dispatcher | | `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `wait4` | | `createCephObjectStore`/`runObjectE2ETestLite`/deletion asserts + zone.json canary | `tests/integration/object/lifecycle` | store create/health/delete helpers, `Sharedstore.Installer()` accessor | | upgrade-suite object usage | stays in upgrade suite | switch to typed clients + `wait4` | diff --git a/tests/integration/object/bucket/lifecycle/lifecycle.go b/tests/integration/object/bucket/lifecycle/lifecycle.go new file mode 100644 index 000000000..6c9fc880e --- /dev/null +++ b/tests/integration/object/bucket/lifecycle/lifecycle.go @@ -0,0 +1,214 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + smithy "github.com/aws/smithy-go" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + rgw "github.com/rook/rook/pkg/operator/ceph/object" + "github.com/rook/rook/tests/framework/utils" + "github.com/rook/rook/tests/integration/object/util/fixture" + "github.com/rook/rook/tests/integration/object/util/obc" + "github.com/rook/rook/tests/integration/object/util/sharedstore" + "github.com/rook/rook/tests/integration/object/util/wait4" +) + +const lifecycle1 = ` +{ + "Rules":[ + { + "ID": "AbortIncompleteMultipartUploads", + "Status": "Enabled", + "Prefix": "", + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 1 + } + } + ] +} +` + +// rules must be sorted by ID to be idempotent +const lifecycle2 = ` +{ + "Rules": [ + { + "ID": "AbortIncompleteMultipartUploads", + "Status": "Enabled", + "Prefix": "", + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 1 + } + }, + { + "ID": "ExpireAfter30Days", + "Status": "Enabled", + "Prefix": "", + "Expiration": { + "Days": 30 + } + } + ] +} +` + +var lifecycleCmpOpts = cmp.Options{ + cmpopts.IgnoreUnexported( + s3types.LifecycleRule{}, + s3types.LifecycleExpiration{}, + s3types.LifecycleRuleFilter{}, + s3types.LifecycleRuleAndOperator{}, + s3types.AbortIncompleteMultipartUpload{}, + s3types.NoncurrentVersionExpiration{}, + s3types.NoncurrentVersionTransition{}, + s3types.Transition{}, + s3types.Tag{}, + ), +} + +// requireLifecycleApplied waits until the bucket's live lifecycle rules match +// the expected JSON configuration. +func requireLifecycleApplied(ctx context.Context, t *testing.T, s3agent *rgw.S3Agent, bucket, expected string) { + t.Helper() + + conf := &s3types.BucketLifecycleConfiguration{} + require.NoError(t, json.Unmarshal([]byte(expected), conf)) + + wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "bucket lifecycle applied", func(ctx context.Context) error { + resp, err := s3agent.Client.GetBucketLifecycleConfiguration(ctx, &s3.GetBucketLifecycleConfigurationInput{Bucket: &bucket}) + if err != nil { + return err + } + if !cmp.Equal(conf.Rules, resp.Rules, lifecycleCmpOpts...) { + return fmt.Errorf("bucket lifecycle rules not in sync: %s", cmp.Diff(conf.Rules, resp.Rules, lifecycleCmpOpts...)) + } + return nil + }) +} + +const Namespace = "test-bucketlifecycle" + +func TestObjectBucketClaimLifecycle(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) { + var ( + defaultName = Namespace + objectStore = store.ObjectStore() + + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + } + + storageClass = obc.StorageClass(defaultName, objectStore) + + bucketName = defaultName + "-obc1" + + obc1 = &bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: bucketName, + Namespace: ns.Name, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: bucketName, + StorageClassName: storageClass.Name, + AdditionalConfig: map[string]string{"bucketLifecycle": lifecycle1}, + }, + } + + obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name) + cephClient = k8sh.RookClientset.CephV1().CephClusters(objectStore.Namespace) + ) + + t.Run("ObjectBucketClaim lifecycle", func(t *testing.T) { + ctx := t.Context() + + fixture.RequireNamespace(t, k8sh, ns) + fixture.RequireStorageClass(t, k8sh, storageClass) + + obc.RequireBound(ctx, t, k8sh, obc1) + s3agent := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc1.Name) + + t.Run(fmt.Sprintf("lifecycle is applied verbatim to bucket %q", bucketName), func(t *testing.T) { + requireLifecycleApplied(ctx, t, s3agent, bucketName, lifecycle1) + }) + + t.Run(fmt.Sprintf("update obc %q bucketLifecycle", obc1.Name), func(t *testing.T) { + obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) { + live.Spec.AdditionalConfig["bucketLifecycle"] = lifecycle2 + }) + }) + + t.Run(fmt.Sprintf("lifecycle update is applied verbatim to bucket %q", bucketName), func(t *testing.T) { + requireLifecycleApplied(ctx, t, s3agent, bucketName, lifecycle2) + }) + + t.Run(fmt.Sprintf("remove obc %q bucketLifecycle", obc1.Name), func(t *testing.T) { + obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) { + live.Spec.AdditionalConfig = map[string]string{} + }) + }) + + t.Run(fmt.Sprintf("lifecycle is removed from bucket %q", bucketName), func(t *testing.T) { + clusters, err := cephClient.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.NotEmpty(t, clusters.Items) + require.NotNil(t, clusters.Items[0].Status.CephVersion) + if strings.HasPrefix(clusters.Items[0].Status.CephVersion.Version, "19.2.3") { + t.Skip("waiting for the rgw fix for the lifecycle removal regression in v19.2.3") + } + + wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "bucket lifecycle removed", func(ctx context.Context) error { + _, err := s3agent.Client.GetBucketLifecycleConfiguration(ctx, &s3.GetBucketLifecycleConfigurationInput{Bucket: &bucketName}) + if err == nil { + return errors.New("bucket lifecycle still present") + } + + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NoSuchLifecycleConfiguration" { + return nil + } + return err + }) + }) + + t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) { + obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc1.Name) + }) + + t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) { + list, err := obcClient.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 0) + }) + }) +}