diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go index 9bed95274..e83dad313 100644 --- a/tests/integration/ceph_object_test.go +++ b/tests/integration/ceph_object_test.go @@ -45,6 +45,7 @@ import ( "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner" + bucketpolicy "github.com/rook/rook/tests/integration/object/bucket/policy" bucketquota "github.com/rook/rook/tests/integration/object/bucket/quota" bucketrw "github.com/rook/rook/tests/integration/object/bucket/rw" "github.com/rook/rook/tests/integration/object/cosi" @@ -215,6 +216,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install sharedObjectStore := sharedstore.Create(s.T(), k8sh, installer, tlsEnable, bucketowner.Namespace, + bucketpolicy.Namespace, bucketquota.Namespace, bucketrw.Namespace, userkeys.Namespace, @@ -227,6 +229,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install defer sharedObjectStore.Destroy() bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, sharedObjectStore) + bucketpolicy.TestObjectBucketClaimPolicy(s.T(), k8sh, sharedObjectStore) bucketquota.TestObjectBucketClaimQuota(s.T(), k8sh, sharedObjectStore) bucketrw.TestObjectBucketClaimReadWrite(s.T(), k8sh, sharedObjectStore) userkeys.TestObjectStoreUserKeys(s.T(), k8sh, sharedObjectStore) @@ -293,248 +296,6 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh logger.Info("OBC, Secret and ConfigMap created") }) - t.Run("OBC bucket policy management", func(t *testing.T) { - bucketName := "bucket-policy-test" - var obName string - var s3client *rgw.S3Agent - bucketPolicy1 := ` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::user/foo" - }, - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject", - "s3:ListBucket", - "s3:GetBucketLocation" - ], - "Resource": [ - "arn:aws:s3:::bucket-policy-test/*" - ] - } - ] -} - ` - - bucketPolicy2 := ` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::user/bar" - }, - "Action": [ - "s3:GetObject", - "s3:ListBucket", - "s3:GetBucketLocation" - ], - "Resource": [ - "arn:aws:s3:::bucket-policy-test/*" - ] - } - ] -} - ` - - t.Run("create obc with bucketPolicy", func(t *testing.T) { - newObc := v1alpha1.ObjectBucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: bucketName, - Namespace: namespace, - }, - Spec: v1alpha1.ObjectBucketClaimSpec{ - BucketName: bucketName, - StorageClassName: bucketStorageClassName, - AdditionalConfig: map[string]string{ - "bucketPolicy": bucketPolicy1, - }, - }, - } - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Create(ctx, &newObc, metav1.CreateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that bucketPolicy is set - assert.Equal(t, bucketPolicy1, obc.Spec.AdditionalConfig["bucketPolicy"]) - - // as the tests are running external to k8s, the internal svc can't be used - labelSelector := "rgw=" + storeName - services, err := k8sh.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) - assert.Nil(t, err) - assert.Equal(t, 1, len(services.Items)) - s3endpoint := services.Items[0].Spec.ClusterIP + ":80" - - secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"]) - s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"]) - - insecure := objectStore.Spec.IsTLSEnabled() - s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil) - assert.Nil(t, err) - logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) - }) - - t.Run("policy was applied verbatim to bucket", func(t *testing.T) { - policyResp, err := s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{ - Bucket: &bucketName, - }) - require.NoError(t, err) - assert.Equal(t, bucketPolicy1, *policyResp.Policy) - }) - - t.Run("update obc bucketPolicy", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - - obc.Spec.AdditionalConfig["bucketPolicy"] = bucketPolicy2 - - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that updated bucketPolicy is set - assert.Equal(t, bucketPolicy2, obc.Spec.AdditionalConfig["bucketPolicy"]) - }) - - t.Run("policy update applied verbatim to bucket", func(t *testing.T) { - var livePolicy string - utils.Retry(20, time.Second, "policy changed", func() bool { - policyResp, err := s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{ - Bucket: &bucketName, - }) - if err != nil { - return false - } - - livePolicy = *policyResp.Policy - return bucketPolicy2 == livePolicy - }) - assert.Equal(t, bucketPolicy2, livePolicy) - }) - - t.Run("remove obc bucketPolicy", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - - obc.Spec.AdditionalConfig = map[string]string{} - - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that bucketPolicy is unset - assert.NotContains(t, obc.Spec.AdditionalConfig, "bucketPolicy") - }) - - t.Run("policy was removed from bucket", func(t *testing.T) { - var err error - utils.Retry(20, time.Second, "policy is gone", func() bool { - _, err = s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{ - Bucket: &bucketName, - }) - return err != nil - }) - require.Error(t, err) - var apiErr smithy.APIError - require.True(t, errors.As(err, &apiErr)) - assert.Equal(t, "NoSuchBucketPolicy", apiErr.ErrorCode()) - }) - - t.Run("delete bucket", func(t *testing.T) { - err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Delete(ctx, bucketName, metav1.DeleteOptions{}) - assert.Nil(t, err) - - absent := utils.Retry(20, time.Second, "OBC is absent", func() bool { - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - return err != nil - }) - assert.True(t, absent) - - absent = utils.Retry(20, time.Second, "OB is absent", func() bool { - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - return err != nil - }) - assert.True(t, absent) - }) - }) - t.Run("OBC bucket lifecycle management", func(t *testing.T) { bucketName := "bucket-lifecycle-test" var obName string diff --git a/tests/integration/object/README.md b/tests/integration/object/README.md index 56852af9c..907a033b1 100644 --- a/tests/integration/object/README.md +++ b/tests/integration/object/README.md @@ -33,6 +33,7 @@ must not collide with std-lib package names (no `io`, no `http`). | test package | operator package | covers | |---|---|---| | `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling | +| `bucket/policy` | `object/bucket` | OBC bucketPolicy management | | `bucket/quota` | `object/bucket` | OBC maxObjects user quota + bucketMaxObjects/bucketMaxSize bucket quota | | `bucket/rw` | `object/bucket` | OBC S3 read/write/delete + OBC-stays-Bound | | `cosi` | `object/cosi` | CephCOSIDriver + COSI bucket provisioning | @@ -41,7 +42,7 @@ must not collide with std-lib package names (no `io`, no `http`). | `user/caps` | `object/user` | user capabilities | | `user/keys` | `object/user` | explicit S3 key management | | `user/opmask` | `object/user` | user op_mask | -| reserved: `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | +| reserved: `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | Shared utilities live under `util/`: @@ -207,7 +208,6 @@ verification; wire the dispatcher; delete the old code; retire | old test | target package(s) | still needs (build in that PR) | |---|---|---| -| `testObjectStoreOperations` bucket policy | `bucket/policy` | — | | `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | move `lifecycleCmpOpts` out of the dispatcher | | `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `wait4` | | `createCephObjectStore`/`runObjectE2ETestLite`/deletion asserts + zone.json canary | `tests/integration/object/lifecycle` | store create/health/delete helpers, `Sharedstore.Installer()` accessor | diff --git a/tests/integration/object/bucket/policy/policy.go b/tests/integration/object/bucket/policy/policy.go new file mode 100644 index 000000000..fdead0889 --- /dev/null +++ b/tests/integration/object/bucket/policy/policy.go @@ -0,0 +1,167 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + smithy "github.com/aws/smithy-go" + bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rook/rook/tests/framework/utils" + "github.com/rook/rook/tests/integration/object/util/fixture" + "github.com/rook/rook/tests/integration/object/util/obc" + "github.com/rook/rook/tests/integration/object/util/sharedstore" + "github.com/rook/rook/tests/integration/object/util/wait4" +) + +// bucketPolicy returns an S3 bucket policy allowing principal the given actions +// on every object in bucket. +func bucketPolicy(bucket, principal string, actions ...string) string { + return fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::user/%s" + }, + "Action": [ + "%s" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, principal, strings.Join(actions, `", + "`), bucket) +} + +const Namespace = "test-bucketpolicy" + +func TestObjectBucketClaimPolicy(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) { + var ( + defaultName = Namespace + objectStore = store.ObjectStore() + + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + } + + storageClass = obc.StorageClass(defaultName, objectStore) + + bucketName = defaultName + "-obc1" + + policy1 = bucketPolicy(bucketName, "foo", + "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket", "s3:GetBucketLocation") + policy2 = bucketPolicy(bucketName, "bar", + "s3:GetObject", "s3:ListBucket", "s3:GetBucketLocation") + + obc1 = &bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: bucketName, + Namespace: ns.Name, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: bucketName, + StorageClassName: storageClass.Name, + AdditionalConfig: map[string]string{"bucketPolicy": policy1}, + }, + } + + obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name) + ) + + t.Run("ObjectBucketClaim policy", func(t *testing.T) { + ctx := t.Context() + + fixture.RequireNamespace(t, k8sh, ns) + fixture.RequireStorageClass(t, k8sh, storageClass) + + obc.RequireBound(ctx, t, k8sh, obc1) + s3agent := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc1.Name) + + t.Run(fmt.Sprintf("policy is applied verbatim to bucket %q", bucketName), func(t *testing.T) { + resp, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName}) + require.NoError(t, err) + require.NotNil(t, resp.Policy) + assert.Equal(t, policy1, *resp.Policy) + }) + + t.Run(fmt.Sprintf("update obc %q bucketPolicy", obc1.Name), func(t *testing.T) { + obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) { + live.Spec.AdditionalConfig["bucketPolicy"] = policy2 + }) + }) + + t.Run(fmt.Sprintf("policy update is applied verbatim to bucket %q", bucketName), func(t *testing.T) { + wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "bucket policy updated", func(ctx context.Context) error { + resp, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName}) + if err != nil { + return err + } + if resp.Policy == nil || *resp.Policy != policy2 { + return errors.New("bucket policy not yet updated") + } + return nil + }) + }) + + t.Run(fmt.Sprintf("remove obc %q bucketPolicy", obc1.Name), func(t *testing.T) { + obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) { + live.Spec.AdditionalConfig = map[string]string{} + }) + }) + + t.Run(fmt.Sprintf("policy is removed from bucket %q", bucketName), func(t *testing.T) { + wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "bucket policy removed", func(ctx context.Context) error { + _, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName}) + if err == nil { + return errors.New("bucket policy still present") + } + + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NoSuchBucketPolicy" { + return nil + } + return err + }) + }) + + t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) { + obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc1.Name) + }) + + t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) { + list, err := obcClient.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 0) + }) + }) +}