diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go index 098c81d1f..9bed95274 100644 --- a/tests/integration/ceph_object_test.go +++ b/tests/integration/ceph_object_test.go @@ -45,6 +45,7 @@ import ( "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner" + bucketquota "github.com/rook/rook/tests/integration/object/bucket/quota" bucketrw "github.com/rook/rook/tests/integration/object/bucket/rw" "github.com/rook/rook/tests/integration/object/cosi" "github.com/rook/rook/tests/integration/object/notification" @@ -214,6 +215,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install sharedObjectStore := sharedstore.Create(s.T(), k8sh, installer, tlsEnable, bucketowner.Namespace, + bucketquota.Namespace, bucketrw.Namespace, userkeys.Namespace, topickafka.Namespace, @@ -225,6 +227,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install defer sharedObjectStore.Destroy() bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, sharedObjectStore) + bucketquota.TestObjectBucketClaimQuota(s.T(), k8sh, sharedObjectStore) bucketrw.TestObjectBucketClaimReadWrite(s.T(), k8sh, sharedObjectStore) userkeys.TestObjectStoreUserKeys(s.T(), k8sh, sharedObjectStore) topickafka.TestBucketTopicKafka(s.T(), k8sh, sharedObjectStore) @@ -290,220 +293,6 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh logger.Info("OBC, Secret and ConfigMap created") }) - t.Run("S3 access to OBC bucket", func(t *testing.T) { - var s3client *rgw.S3Agent - s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName) - s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName) - s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName) - insecure := objectStore.Spec.IsTLSEnabled() - s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil) - - assert.Nil(t, err) - logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) - - t.Run("user quota enforcement", func(t *testing.T) { - _, poErr := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey1, contentType) - assert.Nil(t, poErr) - _, poErr = s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey2, contentType) - assert.Nil(t, poErr) - logger.Infof("Testing the max object limit") - quotaEnforced := utils.Retry(30, 2*time.Second, "user quota enforced", func() bool { - _, err := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey3, contentType) - if err != nil { - return true - } - // delete so next attempt creates a new object rather than overwriting - s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey3) //nolint:errcheck - return false - }) - assert.True(t, quotaEnforced) - }) - - t.Run("update user quota limits", func(t *testing.T) { - poErr := helper.BucketClient.UpdateObc(obcName, bucketStorageClassName, bucketname, newMaxObject, true) - assert.Nil(t, poErr) - updated := utils.Retry(20, 2*time.Second, "OBC is updated", func() bool { - return helper.BucketClient.CheckOBMaxObject(obcName, newMaxObject) - }) - assert.True(t, updated) - logger.Infof("Testing the updated object limit") - _, poErr = s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey3, contentType) - assert.NoError(t, poErr) - quotaEnforced := utils.Retry(30, 2*time.Second, "updated user quota enforced", func() bool { - _, putErr := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey4, contentType) - if putErr != nil { - return true - } - // delete so next attempt creates a new object rather than overwriting - s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey4) //nolint:errcheck - return false - }) - assert.True(t, quotaEnforced) - }) - - t.Run("delete objects", func(t *testing.T) { - _, delobjErr := s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey1) - assert.Nil(t, delobjErr) - _, delobjErr = s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey2) - assert.Nil(t, delobjErr) - _, delobjErr = s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey3) - assert.Nil(t, delobjErr) - logger.Info("Objects deleted on bucket successfully") - }) - }) - - // this test deviates from the others in this package in that it does not - // rely on kubectl and uses the k8s api directly - t.Run("OBC bucket quota enforcement", func(t *testing.T) { - bucketName := "bucket-quota-test" - var obName string - var s3client *rgw.S3Agent - - t.Run("create obc with bucketMaxObjects", func(t *testing.T) { - newObc := v1alpha1.ObjectBucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: bucketName, - Namespace: namespace, - }, - Spec: v1alpha1.ObjectBucketClaimSpec{ - BucketName: bucketName, - StorageClassName: bucketStorageClassName, - AdditionalConfig: map[string]string{ - "bucketMaxObjects": "1", - }, - }, - } - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Create(ctx, &newObc, metav1.CreateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that bucketMaxObjects is set - assert.True(t, obc.Spec.AdditionalConfig["bucketMaxObjects"] == "1") - - // as the tests are running external to k8s, the internal svc can't be used - labelSelector := "rgw=" + storeName - services, err := k8sh.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) - assert.Nil(t, err) - assert.Equal(t, 1, len(services.Items)) - s3endpoint := services.Items[0].Spec.ClusterIP + ":80" - - secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"]) - s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"]) - - insecure := objectStore.Spec.IsTLSEnabled() - s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil) - assert.Nil(t, err) - logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) - }) - - t.Run("bucketMaxObjects quota is enforced", func(t *testing.T) { - // first object should succeed - _, err := s3client.PutObjectInBucket(ctx, bucketName, ObjBody, ObjectKey1, contentType) - assert.Nil(t, err) - // second object should fail as bucket quota is 1 - _, err = s3client.PutObjectInBucket(ctx, bucketName, ObjBody, ObjectKey2, contentType) - assert.Error(t, err) - // cleanup bucket - _, err = s3client.DeleteObjectInBucket(ctx, bucketName, ObjectKey1) - assert.Nil(t, err) - _, err = s3client.DeleteObjectInBucket(ctx, bucketName, ObjectKey2) - assert.Nil(t, err) - }) - - t.Run("change quota to bucketMaxSize", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - - obc.Spec.AdditionalConfig = map[string]string{"bucketMaxSize": "4Ki"} - - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool { - liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound - }) - assert.True(t, obcBound) - - // wait until obc is Bound to lookup the ob name - obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - assert.Nil(t, err) - obName = obc.Spec.ObjectBucketName - - obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool { - liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - if err != nil { - return false - } - - return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound - }) - assert.True(t, obBound) - - // verify that bucketMaxSize is set - assert.True(t, obc.Spec.AdditionalConfig["bucketMaxSize"] == "4Ki") - }) - - t.Run("bucketMaxSize quota is enforced", func(t *testing.T) { - // first ~3KiB Object should succeed - _, err := s3client.PutObjectInBucket(ctx, bucketName, strings.Repeat("1", 3072), ObjectKey1, contentType) - assert.Nil(t, err) - // second ~2KiB Object should fail as bucket quota is 4KiB - _, err = s3client.PutObjectInBucket(ctx, bucketName, strings.Repeat("2", 2048), ObjectKey2, contentType) - assert.Error(t, err) - // cleanup bucket - _, err = s3client.DeleteObjectInBucket(ctx, bucketName, ObjectKey1) - assert.Nil(t, err) - _, err = s3client.DeleteObjectInBucket(ctx, bucketName, ObjectKey2) - assert.Nil(t, err) - }) - - t.Run("delete bucket", func(t *testing.T) { - err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Delete(ctx, bucketName, metav1.DeleteOptions{}) - assert.Nil(t, err) - - absent := utils.Retry(20, 2*time.Second, "OBC is absent", func() bool { - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{}) - return err != nil - }) - assert.True(t, absent) - - absent = utils.Retry(20, 2*time.Second, "OB is absent", func() bool { - _, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{}) - return err != nil - }) - assert.True(t, absent) - }) - }) - t.Run("OBC bucket policy management", func(t *testing.T) { bucketName := "bucket-policy-test" var obName string diff --git a/tests/integration/object/README.md b/tests/integration/object/README.md index 2b86c6764..56852af9c 100644 --- a/tests/integration/object/README.md +++ b/tests/integration/object/README.md @@ -33,6 +33,7 @@ must not collide with std-lib package names (no `io`, no `http`). | test package | operator package | covers | |---|---|---| | `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling | +| `bucket/quota` | `object/bucket` | OBC maxObjects user quota + bucketMaxObjects/bucketMaxSize bucket quota | | `bucket/rw` | `object/bucket` | OBC S3 read/write/delete + OBC-stays-Bound | | `cosi` | `object/cosi` | CephCOSIDriver + COSI bucket provisioning | | `notification` | `object/notification` | CephBucketNotification HTTP endpoint delivery | @@ -40,7 +41,7 @@ must not collide with std-lib package names (no `io`, no `http`). | `user/caps` | `object/user` | user capabilities | | `user/keys` | `object/user` | explicit S3 key management | | `user/opmask` | `object/user` | user op_mask | -| reserved: `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | +| reserved: `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | Shared utilities live under `util/`: @@ -206,7 +207,6 @@ verification; wire the dispatcher; delete the old code; retire | old test | target package(s) | still needs (build in that PR) | |---|---|---| -| `testObjectStoreOperations` user/bucket quotas | `bucket/quota` | — | | `testObjectStoreOperations` bucket policy | `bucket/policy` | — | | `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | move `lifecycleCmpOpts` out of the dispatcher | | `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `wait4` | diff --git a/tests/integration/object/bucket/quota/quota.go b/tests/integration/object/bucket/quota/quota.go new file mode 100644 index 000000000..86a025163 --- /dev/null +++ b/tests/integration/object/bucket/quota/quota.go @@ -0,0 +1,215 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "context" + "fmt" + "strings" + "testing" + + bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + rgw "github.com/rook/rook/pkg/operator/ceph/object" + "github.com/rook/rook/tests/framework/utils" + "github.com/rook/rook/tests/integration/object/util/fixture" + "github.com/rook/rook/tests/integration/object/util/obc" + "github.com/rook/rook/tests/integration/object/util/sharedstore" + "github.com/rook/rook/tests/integration/object/util/wait4" +) + +const ( + body = "test bucket quota payload" + contentType = "text/plain" + + key1 = "obj1" + key2 = "obj2" + key3 = "obj3" + key4 = "obj4" +) + +// obAdditionalConfig returns a predicate matching an ObjectBucket whose endpoint +// additionalConfig has key set to value, the sign that the provisioner has applied +// a quota change to the backing bucket. +func obAdditionalConfig(key, value string) func(*bktv1alpha1.ObjectBucket) bool { + return func(ob *bktv1alpha1.ObjectBucket) bool { + if ob.Spec.Connection == nil || ob.Spec.Connection.Endpoint == nil { + return false + } + return ob.Spec.Connection.Endpoint.AdditionalConfigData[key] == value + } +} + +// requireQuotaEnforced waits until putting key into bucket is rejected. rgw +// enforces user quota against per-instance cached stats, so enforcement can lag +// recent writes; an unexpectedly-successful put is deleted so the next attempt +// creates a new object rather than overwriting. +func requireQuotaEnforced(ctx context.Context, t *testing.T, s3 *rgw.S3Agent, bucket, key string) { + t.Helper() + + wait4.RequireEventually(ctx, t, wait4.TimeoutMedium, fmt.Sprintf("quota rejects object %q", key), func(ctx context.Context) error { + _, err := s3.PutObjectInBucket(ctx, bucket, body, key, contentType) + if err == nil { + s3.DeleteObjectInBucket(ctx, bucket, key) //nolint:errcheck + return fmt.Errorf("quota not yet enforced: put %q succeeded", key) + } + return nil + }) +} + +const Namespace = "test-bucketquota" + +func TestObjectBucketClaimQuota(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) { + var ( + defaultName = Namespace + objectStore = store.ObjectStore() + + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + } + + storageClass = obc.StorageClass(defaultName, objectStore) + + obc1 = &bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-obc1", + Namespace: ns.Name, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: defaultName + "-obc1", + StorageClassName: storageClass.Name, + AdditionalConfig: map[string]string{"maxObjects": "2"}, + }, + } + + obc2 = &bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-obc2", + Namespace: ns.Name, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: defaultName + "-obc2", + StorageClassName: storageClass.Name, + AdditionalConfig: map[string]string{"bucketMaxObjects": "1"}, + }, + } + + obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name) + obClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets() + ) + + t.Run("ObjectBucketClaim quota", func(t *testing.T) { + ctx := t.Context() + + fixture.RequireNamespace(t, k8sh, ns) + fixture.RequireStorageClass(t, k8sh, storageClass) + + liveObc1 := obc.RequireBound(ctx, t, k8sh, obc1) + s3a := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc1.Name) + + t.Run(fmt.Sprintf("maxObjects quota is enforced on obc %q", obc1.Name), func(t *testing.T) { + _, err := s3a.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key1, contentType) + require.NoError(t, err) + _, err = s3a.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key2, contentType) + require.NoError(t, err) + + requireQuotaEnforced(ctx, t, s3a, obc1.Spec.BucketName, key3) + }) + + t.Run(fmt.Sprintf("raise maxObjects on obc %q", obc1.Name), func(t *testing.T) { + obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) { + live.Spec.AdditionalConfig["maxObjects"] = "3" + }) + + wait4.RequireCondition(ctx, t, obClient, liveObc1.Spec.ObjectBucketName, + obAdditionalConfig("maxObjects", "3"), wait4.TimeoutShort) + }) + + t.Run(fmt.Sprintf("raised maxObjects quota admits object %q", key3), func(t *testing.T) { + _, err := s3a.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key3, contentType) + require.NoError(t, err) + + requireQuotaEnforced(ctx, t, s3a, obc1.Spec.BucketName, key4) + }) + + t.Run(fmt.Sprintf("delete objects in bucket %q", obc1.Spec.BucketName), func(t *testing.T) { + for _, key := range []string{key1, key2, key3} { + _, err := s3a.DeleteObjectInBucket(ctx, obc1.Spec.BucketName, key) + assert.NoError(t, err) + } + }) + + t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) { + obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc1.Name) + }) + + liveObc2 := obc.RequireBound(ctx, t, k8sh, obc2) + s3b := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc2.Name) + + t.Run(fmt.Sprintf("bucketMaxObjects quota is enforced on obc %q", obc2.Name), func(t *testing.T) { + _, err := s3b.PutObjectInBucket(ctx, obc2.Spec.BucketName, body, key1, contentType) + require.NoError(t, err) + + // unlike the rgw-cached user quota, bucket quota applies immediately + _, err = s3b.PutObjectInBucket(ctx, obc2.Spec.BucketName, body, key2, contentType) + assert.Error(t, err) + + for _, key := range []string{key1, key2} { + _, err := s3b.DeleteObjectInBucket(ctx, obc2.Spec.BucketName, key) + assert.NoError(t, err) + } + }) + + t.Run(fmt.Sprintf("switch obc %q to a bucketMaxSize quota", obc2.Name), func(t *testing.T) { + obc.Update(ctx, t, k8sh, ns.Name, obc2.Name, func(live *bktv1alpha1.ObjectBucketClaim) { + live.Spec.AdditionalConfig = map[string]string{"bucketMaxSize": "4Ki"} + }) + + wait4.RequireCondition(ctx, t, obClient, liveObc2.Spec.ObjectBucketName, + obAdditionalConfig("bucketMaxSize", "4Ki"), wait4.TimeoutShort) + }) + + t.Run(fmt.Sprintf("bucketMaxSize quota is enforced on obc %q", obc2.Name), func(t *testing.T) { + _, err := s3b.PutObjectInBucket(ctx, obc2.Spec.BucketName, strings.Repeat("1", 3072), key1, contentType) + require.NoError(t, err) + + _, err = s3b.PutObjectInBucket(ctx, obc2.Spec.BucketName, strings.Repeat("2", 2048), key2, contentType) + assert.Error(t, err) + + for _, key := range []string{key1, key2} { + _, err := s3b.DeleteObjectInBucket(ctx, obc2.Spec.BucketName, key) + assert.NoError(t, err) + } + }) + + t.Run(fmt.Sprintf("delete obc %q", obc2.Name), func(t *testing.T) { + obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc2.Name) + }) + + t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) { + list, err := obcClient.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 0) + }) + }) +}