From 2cf5008bc8e0945c82eb1c2accf15ea9f42a47a6 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Thu, 2 Jul 2026 12:12:29 -0700 Subject: [PATCH] test: convert the object bucket policy suite to the shared store Add tests/integration/object/bucket/policy, converting the bucket policy slice of the legacy testObjectStoreOperations onto the shared store: an OBC created with a bucketPolicy additionalConfig, the policy verified verbatim on the rgw bucket over the S3 API, then updated and removed with the bucket polled until it reflects each change. The policy JSON is generated from the bucket name, which the legacy test hardcoded in the Resource ARN. It runs against the shared store in both the TLS and non-TLS passes. Drop the converted policy block from the legacy test. Signed-off-by: Joshua Hoblitt --- tests/integration/ceph_object_test.go | 245 +----------------- tests/integration/object/README.md | 4 +- .../object/bucket/policy/policy.go | 167 ++++++++++++ 3 files changed, 172 insertions(+), 244 deletions(-) create mode 100644 tests/integration/object/bucket/policy/policy.go diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go index 9bed95274..e83dad313 100644 --- a/tests/integration/ceph_object_test.go +++ b/tests/integration/ceph_object_test.go @@ -45,6 +45,7 @@ import ( "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner" + bucketpolicy "github.com/rook/rook/tests/integration/object/bucket/policy" bucketquota "github.com/rook/rook/tests/integration/object/bucket/quota" bucketrw "github.com/rook/rook/tests/integration/object/bucket/rw" "github.com/rook/rook/tests/integration/object/cosi" @@ -215,6 +216,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install sharedObjectStore := sharedstore.Create(s.T(), k8sh, installer, tlsEnable, bucketowner.Namespace, + bucketpolicy.Namespace, bucketquota.Namespace, bucketrw.Namespace, userkeys.Namespace, @@ -227,6 +229,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install defer sharedObjectStore.Destroy() bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, sharedObjectStore) + bucketpolicy.TestObjectBucketClaimPolicy(s.T(), k8sh, sharedObjectStore) bucketquota.TestObjectBucketClaimQuota(s.T(), k8sh, sharedObjectStore) bucketrw.TestObjectBucketClaimReadWrite(s.T(), k8sh, sharedObjectStore) userkeys.TestObjectStoreUserKeys(s.T(), k8sh, sharedObjectStore) @@ -293,248 +296,6 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh logger.Info("OBC, Secret and ConfigMap created") }) - t.Run("OBC bucket policy management", func(t *testing.T) { - bucketName := "bucket-policy-test" - var obName string - var s3client *rgw.S3Agent - bucketPolicy1 := ` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::user/foo" - }, - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject", - "s3:ListBucket", - "s3:GetBucketLocation" - ], - "Resource": [ - "arn:aws:s3:::bucket-policy-test/*" - ] - } - ] -} - ` - - bucketPolicy2 := ` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::user/bar" - }, - "Action": [ - "s3:GetObject", - "s3:ListBucket", - "s3:GetBucketLocation" - ], - "Resource": [ - "arn:aws:s3:::bucket-policy-test/*" - ] - } - ] -} - ` - - t.Run("create obc with bucketPolicy", func(t *testing.T) { - newObc := v1alpha1.ObjectBucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: bucketName, - Namespace: namespace, - }, - Spec: v1alpha1.ObjectBucketClaimSpec{ - BucketName: bucketName, - StorageClassName: bucketStorageClassName, - AdditionalConfig: map[string]string{ - "bucketPolicy": bucketPolicy1, - }, - }, - } - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Create(ctx, &newObc, metav1.CreateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that bucketPolicy is set - assert.Equal(t, bucketPolicy1, obc.Spec.AdditionalConfig["bucketPolicy"]) - - // as the tests are running external to k8s, the internal svc can't be used - labelSelector := "rgw=" + storeName - services, err := k8sh.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) - assert.Nil(t, err) - assert.Equal(t, 1, len(services.Items)) - s3endpoint := services.Items[0].Spec.ClusterIP + ":80" - - secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"]) - s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"]) - - insecure := objectStore.Spec.IsTLSEnabled() - s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil) - assert.Nil(t, err) - logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) - }) - - t.Run("policy was applied verbatim to bucket", func(t *testing.T) { - policyResp, err := s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{ - Bucket: &bucketName, - }) - require.NoError(t, err) - assert.Equal(t, bucketPolicy1, *policyResp.Policy) - }) - - t.Run("update obc bucketPolicy", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - - obc.Spec.AdditionalConfig["bucketPolicy"] = bucketPolicy2 - - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that updated bucketPolicy is set - assert.Equal(t, bucketPolicy2, obc.Spec.AdditionalConfig["bucketPolicy"]) - }) - - t.Run("policy update applied verbatim to bucket", func(t *testing.T) { - var livePolicy string - utils.Retry(20, time.Second, "policy changed", func() bool { - policyResp, err := s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{ - Bucket: &bucketName, - }) - if err != nil { - return false - } - - livePolicy = *policyResp.Policy - return bucketPolicy2 == livePolicy - }) - assert.Equal(t, bucketPolicy2, livePolicy) - }) - - t.Run("remove obc bucketPolicy", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - - obc.Spec.AdditionalConfig = map[string]string{} - - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that bucketPolicy is unset - assert.NotContains(t, obc.Spec.AdditionalConfig, "bucketPolicy") - }) - - t.Run("policy was removed from bucket", func(t *testing.T) { - var err error - utils.Retry(20, time.Second, "policy is gone", func() bool { - _, err = s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{ - Bucket: &bucketName, - }) - return err != nil - }) - require.Error(t, err) - var apiErr smithy.APIError - require.True(t, errors.As(err, &apiErr)) - assert.Equal(t, "NoSuchBucketPolicy", apiErr.ErrorCode()) - }) - - t.Run("delete bucket", func(t *testing.T) { - err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Delete(ctx, bucketName, metav1.DeleteOptions{}) - assert.Nil(t, err) - - absent := utils.Retry(20, time.Second, "OBC is absent", func() bool { - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - return err != nil - }) - assert.True(t, absent) - - absent = utils.Retry(20, time.Second, "OB is absent", func() bool { - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - return err != nil - }) - assert.True(t, absent) - }) - }) - t.Run("OBC bucket lifecycle management", func(t *testing.T) { bucketName := "bucket-lifecycle-test" var obName string diff --git a/tests/integration/object/README.md b/tests/integration/object/README.md index 56852af9c..907a033b1 100644 --- a/tests/integration/object/README.md +++ b/tests/integration/object/README.md @@ -33,6 +33,7 @@ must not collide with std-lib package names (no `io`, no `http`). | test package | operator package | covers | |---|---|---| | `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling | +| `bucket/policy` | `object/bucket` | OBC bucketPolicy management | | `bucket/quota` | `object/bucket` | OBC maxObjects user quota + bucketMaxObjects/bucketMaxSize bucket quota | | `bucket/rw` | `object/bucket` | OBC S3 read/write/delete + OBC-stays-Bound | | `cosi` | `object/cosi` | CephCOSIDriver + COSI bucket provisioning | @@ -41,7 +42,7 @@ must not collide with std-lib package names (no `io`, no `http`). | `user/caps` | `object/user` | user capabilities | | `user/keys` | `object/user` | explicit S3 key management | | `user/opmask` | `object/user` | user op_mask | -| reserved: `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | +| reserved: `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | Shared utilities live under `util/`: @@ -207,7 +208,6 @@ verification; wire the dispatcher; delete the old code; retire | old test | target package(s) | still needs (build in that PR) | |---|---|---| -| `testObjectStoreOperations` bucket policy | `bucket/policy` | — | | `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | move `lifecycleCmpOpts` out of the dispatcher | | `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `wait4` | | `createCephObjectStore`/`runObjectE2ETestLite`/deletion asserts + zone.json canary | `tests/integration/object/lifecycle` | store create/health/delete helpers, `Sharedstore.Installer()` accessor | diff --git a/tests/integration/object/bucket/policy/policy.go b/tests/integration/object/bucket/policy/policy.go new file mode 100644 index 000000000..fdead0889 --- /dev/null +++ b/tests/integration/object/bucket/policy/policy.go @@ -0,0 +1,167 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + smithy "github.com/aws/smithy-go" + bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rook/rook/tests/framework/utils" + "github.com/rook/rook/tests/integration/object/util/fixture" + "github.com/rook/rook/tests/integration/object/util/obc" + "github.com/rook/rook/tests/integration/object/util/sharedstore" + "github.com/rook/rook/tests/integration/object/util/wait4" +) + +// bucketPolicy returns an S3 bucket policy allowing principal the given actions +// on every object in bucket. +func bucketPolicy(bucket, principal string, actions ...string) string { + return fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::user/%s" + }, + "Action": [ + "%s" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, principal, strings.Join(actions, `", + "`), bucket) +} + +const Namespace = "test-bucketpolicy" + +func TestObjectBucketClaimPolicy(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) { + var ( + defaultName = Namespace + objectStore = store.ObjectStore() + + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + } + + storageClass = obc.StorageClass(defaultName, objectStore) + + bucketName = defaultName + "-obc1" + + policy1 = bucketPolicy(bucketName, "foo", + "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket", "s3:GetBucketLocation") + policy2 = bucketPolicy(bucketName, "bar", + "s3:GetObject", "s3:ListBucket", "s3:GetBucketLocation") + + obc1 = &bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: bucketName, + Namespace: ns.Name, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: bucketName, + StorageClassName: storageClass.Name, + AdditionalConfig: map[string]string{"bucketPolicy": policy1}, + }, + } + + obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name) + ) + + t.Run("ObjectBucketClaim policy", func(t *testing.T) { + ctx := t.Context() + + fixture.RequireNamespace(t, k8sh, ns) + fixture.RequireStorageClass(t, k8sh, storageClass) + + obc.RequireBound(ctx, t, k8sh, obc1) + s3agent := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc1.Name) + + t.Run(fmt.Sprintf("policy is applied verbatim to bucket %q", bucketName), func(t *testing.T) { + resp, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName}) + require.NoError(t, err) + require.NotNil(t, resp.Policy) + assert.Equal(t, policy1, *resp.Policy) + }) + + t.Run(fmt.Sprintf("update obc %q bucketPolicy", obc1.Name), func(t *testing.T) { + obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) { + live.Spec.AdditionalConfig["bucketPolicy"] = policy2 + }) + }) + + t.Run(fmt.Sprintf("policy update is applied verbatim to bucket %q", bucketName), func(t *testing.T) { + wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "bucket policy updated", func(ctx context.Context) error { + resp, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName}) + if err != nil { + return err + } + if resp.Policy == nil || *resp.Policy != policy2 { + return errors.New("bucket policy not yet updated") + } + return nil + }) + }) + + t.Run(fmt.Sprintf("remove obc %q bucketPolicy", obc1.Name), func(t *testing.T) { + obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) { + live.Spec.AdditionalConfig = map[string]string{} + }) + }) + + t.Run(fmt.Sprintf("policy is removed from bucket %q", bucketName), func(t *testing.T) { + wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "bucket policy removed", func(ctx context.Context) error { + _, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName}) + if err == nil { + return errors.New("bucket policy still present") + } + + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NoSuchBucketPolicy" { + return nil + } + return err + }) + }) + + t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) { + obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc1.Name) + }) + + t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) { + list, err := obcClient.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 0) + }) + }) +}