Merge pull request #17884 from jhoblitt/convert-bucket-policy-to-sharedstore

test: convert the object bucket policy suite to the shared store
This commit is contained in:
Joshua Hoblitt authored and GitHub committed 2026-07-13 13:08:59 -07:00
commit 64973c65d5
3 files changed
+172 -244

No files matched your search

+3 -242
View File
@@ -45,6 +45,7 @@ import (
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner"
bucketpolicy "github.com/rook/rook/tests/integration/object/bucket/policy"
bucketquota "github.com/rook/rook/tests/integration/object/bucket/quota"
bucketrw "github.com/rook/rook/tests/integration/object/bucket/rw"
"github.com/rook/rook/tests/integration/object/cosi"
@@ -215,6 +216,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
sharedObjectStore := sharedstore.Create(s.T(), k8sh, installer, tlsEnable,
bucketowner.Namespace,
bucketpolicy.Namespace,
bucketquota.Namespace,
bucketrw.Namespace,
userkeys.Namespace,
@@ -227,6 +229,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
defer sharedObjectStore.Destroy()
bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, sharedObjectStore)
bucketpolicy.TestObjectBucketClaimPolicy(s.T(), k8sh, sharedObjectStore)
bucketquota.TestObjectBucketClaimQuota(s.T(), k8sh, sharedObjectStore)
bucketrw.TestObjectBucketClaimReadWrite(s.T(), k8sh, sharedObjectStore)
userkeys.TestObjectStoreUserKeys(s.T(), k8sh, sharedObjectStore)
@@ -293,248 +296,6 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
logger.Info("OBC, Secret and ConfigMap created")
})
t.Run("OBC bucket policy management", func(t *testing.T) {
bucketName := "bucket-policy-test"
var obName string
var s3client *rgw.S3Agent
bucketPolicy1 := `
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam:::user/foo"
},
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::bucket-policy-test/*"
]
}
]
}
`
bucketPolicy2 := `
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam:::user/bar"
},
"Action": [
"s3:GetObject",
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::bucket-policy-test/*"
]
}
]
}
`
t.Run("create obc with bucketPolicy", func(t *testing.T) {
newObc := v1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: bucketName,
Namespace: namespace,
},
Spec: v1alpha1.ObjectBucketClaimSpec{
BucketName: bucketName,
StorageClassName: bucketStorageClassName,
AdditionalConfig: map[string]string{
"bucketPolicy": bucketPolicy1,
},
},
}
_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Create(ctx, &newObc, metav1.CreateOptions{})
assert.Nil(t, err)
obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool {
liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
if err != nil {
return false
}
return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound
})
assert.True(t, obcBound)
// wait until obc is Bound to lookup the ob name
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
obName = obc.Spec.ObjectBucketName
obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool {
liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
if err != nil {
return false
}
return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound
})
assert.True(t, obBound)
// verify that bucketPolicy is set
assert.Equal(t, bucketPolicy1, obc.Spec.AdditionalConfig["bucketPolicy"])
// as the tests are running external to k8s, the internal svc can't be used
labelSelector := "rgw=" + storeName
services, err := k8sh.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
assert.Nil(t, err)
assert.Equal(t, 1, len(services.Items))
s3endpoint := services.Items[0].Spec.ClusterIP + ":80"
secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"])
s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"])
insecure := objectStore.Spec.IsTLSEnabled()
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
assert.Nil(t, err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
})
t.Run("policy was applied verbatim to bucket", func(t *testing.T) {
policyResp, err := s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
Bucket: &bucketName,
})
require.NoError(t, err)
assert.Equal(t, bucketPolicy1, *policyResp.Policy)
})
t.Run("update obc bucketPolicy", func(t *testing.T) {
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
obc.Spec.AdditionalConfig["bucketPolicy"] = bucketPolicy2
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{})
assert.Nil(t, err)
obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool {
liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
if err != nil {
return false
}
return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound
})
assert.True(t, obcBound)
// wait until obc is Bound to lookup the ob name
obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
obName = obc.Spec.ObjectBucketName
obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool {
liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
if err != nil {
return false
}
return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound
})
assert.True(t, obBound)
// verify that updated bucketPolicy is set
assert.Equal(t, bucketPolicy2, obc.Spec.AdditionalConfig["bucketPolicy"])
})
t.Run("policy update applied verbatim to bucket", func(t *testing.T) {
var livePolicy string
utils.Retry(20, time.Second, "policy changed", func() bool {
policyResp, err := s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
Bucket: &bucketName,
})
if err != nil {
return false
}
livePolicy = *policyResp.Policy
return bucketPolicy2 == livePolicy
})
assert.Equal(t, bucketPolicy2, livePolicy)
})
t.Run("remove obc bucketPolicy", func(t *testing.T) {
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
obc.Spec.AdditionalConfig = map[string]string{}
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{})
assert.Nil(t, err)
obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool {
liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
if err != nil {
return false
}
return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound
})
assert.True(t, obcBound)
// wait until obc is Bound to lookup the ob name
obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
obName = obc.Spec.ObjectBucketName
obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool {
liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
if err != nil {
return false
}
return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound
})
assert.True(t, obBound)
// verify that bucketPolicy is unset
assert.NotContains(t, obc.Spec.AdditionalConfig, "bucketPolicy")
})
t.Run("policy was removed from bucket", func(t *testing.T) {
var err error
utils.Retry(20, time.Second, "policy is gone", func() bool {
_, err = s3client.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
Bucket: &bucketName,
})
return err != nil
})
require.Error(t, err)
var apiErr smithy.APIError
require.True(t, errors.As(err, &apiErr))
assert.Equal(t, "NoSuchBucketPolicy", apiErr.ErrorCode())
})
t.Run("delete bucket", func(t *testing.T) {
err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Delete(ctx, bucketName, metav1.DeleteOptions{})
assert.Nil(t, err)
absent := utils.Retry(20, time.Second, "OBC is absent", func() bool {
_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
return err != nil
})
assert.True(t, absent)
absent = utils.Retry(20, time.Second, "OB is absent", func() bool {
_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
return err != nil
})
assert.True(t, absent)
})
})
t.Run("OBC bucket lifecycle management", func(t *testing.T) {
bucketName := "bucket-lifecycle-test"
var obName string
+2 -2
View File
@@ -33,6 +33,7 @@ must not collide with std-lib package names (no `io`, no `http`).
| test package | operator package | covers |
|---|---|---|
| `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling |
| `bucket/policy` | `object/bucket` | OBC bucketPolicy management |
| `bucket/quota` | `object/bucket` | OBC maxObjects user quota + bucketMaxObjects/bucketMaxSize bucket quota |
| `bucket/rw` | `object/bucket` | OBC S3 read/write/delete + OBC-stays-Bound |
| `cosi` | `object/cosi` | CephCOSIDriver + COSI bucket provisioning |
@@ -41,7 +42,7 @@ must not collide with std-lib package names (no `io`, no `http`).
| `user/caps` | `object/user` | user capabilities |
| `user/keys` | `object/user` | explicit S3 key management |
| `user/opmask` | `object/user` | user op_mask |
| reserved: `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions |
| reserved: `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions |
Shared utilities live under `util/`:
@@ -207,7 +208,6 @@ verification; wire the dispatcher; delete the old code; retire
| old test | target package(s) | still needs (build in that PR) |
|---|---|---|
| `testObjectStoreOperations` bucket policy | `bucket/policy` | — |
| `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | move `lifecycleCmpOpts` out of the dispatcher |
| `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `wait4` |
| `createCephObjectStore`/`runObjectE2ETestLite`/deletion asserts + zone.json canary | `tests/integration/object/lifecycle` | store create/health/delete helpers, `Sharedstore.Installer()` accessor |
@@ -0,0 +1,167 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package policy
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3"
smithy "github.com/aws/smithy-go"
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rook/rook/tests/framework/utils"
"github.com/rook/rook/tests/integration/object/util/fixture"
"github.com/rook/rook/tests/integration/object/util/obc"
"github.com/rook/rook/tests/integration/object/util/sharedstore"
"github.com/rook/rook/tests/integration/object/util/wait4"
)
// bucketPolicy returns an S3 bucket policy allowing principal the given actions
// on every object in bucket.
func bucketPolicy(bucket, principal string, actions ...string) string {
return fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam:::user/%s"
},
"Action": [
"%s"
],
"Resource": [
"arn:aws:s3:::%s/*"
]
}
]
}`, principal, strings.Join(actions, `",
"`), bucket)
}
const Namespace = "test-bucketpolicy"
func TestObjectBucketClaimPolicy(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
var (
defaultName = Namespace
objectStore = store.ObjectStore()
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
}
storageClass = obc.StorageClass(defaultName, objectStore)
bucketName = defaultName + "-obc1"
policy1 = bucketPolicy(bucketName, "foo",
"s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket", "s3:GetBucketLocation")
policy2 = bucketPolicy(bucketName, "bar",
"s3:GetObject", "s3:ListBucket", "s3:GetBucketLocation")
obc1 = &bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: bucketName,
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: bucketName,
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{"bucketPolicy": policy1},
},
}
obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name)
)
t.Run("ObjectBucketClaim policy", func(t *testing.T) {
ctx := t.Context()
fixture.RequireNamespace(t, k8sh, ns)
fixture.RequireStorageClass(t, k8sh, storageClass)
obc.RequireBound(ctx, t, k8sh, obc1)
s3agent := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc1.Name)
t.Run(fmt.Sprintf("policy is applied verbatim to bucket %q", bucketName), func(t *testing.T) {
resp, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName})
require.NoError(t, err)
require.NotNil(t, resp.Policy)
assert.Equal(t, policy1, *resp.Policy)
})
t.Run(fmt.Sprintf("update obc %q bucketPolicy", obc1.Name), func(t *testing.T) {
obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) {
live.Spec.AdditionalConfig["bucketPolicy"] = policy2
})
})
t.Run(fmt.Sprintf("policy update is applied verbatim to bucket %q", bucketName), func(t *testing.T) {
wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "bucket policy updated", func(ctx context.Context) error {
resp, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName})
if err != nil {
return err
}
if resp.Policy == nil || *resp.Policy != policy2 {
return errors.New("bucket policy not yet updated")
}
return nil
})
})
t.Run(fmt.Sprintf("remove obc %q bucketPolicy", obc1.Name), func(t *testing.T) {
obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) {
live.Spec.AdditionalConfig = map[string]string{}
})
})
t.Run(fmt.Sprintf("policy is removed from bucket %q", bucketName), func(t *testing.T) {
wait4.RequireEventually(ctx, t, wait4.TimeoutShort, "bucket policy removed", func(ctx context.Context) error {
_, err := s3agent.Client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{Bucket: &bucketName})
if err == nil {
return errors.New("bucket policy still present")
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NoSuchBucketPolicy" {
return nil
}
return err
})
})
t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) {
obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc1.Name)
})
t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) {
list, err := obcClient.List(ctx, metav1.ListOptions{})
require.NoError(t, err)
assert.Len(t, list.Items, 0)
})
})
}