test: convert the object bucket quota suite to the shared store

Add tests/integration/object/bucket/quota, covering both quota behaviors
from the legacy testObjectStoreOperations on OBCs of its own: user quota
via the OBC maxObjects additionalConfig (enforced, then raised and
re-enforced, synced on the ObjectBucket reflecting the new limit) and
bucket quota via bucketMaxObjects switched to bucketMaxSize. It runs
against the shared store in both the TLS and non-TLS passes.

Drop the converted S3-access and bucket-quota blocks from the legacy test;
the main OBC now exists only as a dependents fixture for the store
deletion checks.

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
This commit is contained in:
Joshua Hoblitt committed 2026-07-06 10:35:37 -07:00
1 parent 8dfc5d408d
commit 84c10661da
3 files changed
+220 -216

No files matched your search

+3 -214
View File
@@ -45,6 +45,7 @@ import (
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner"
bucketquota "github.com/rook/rook/tests/integration/object/bucket/quota"
bucketrw "github.com/rook/rook/tests/integration/object/bucket/rw"
"github.com/rook/rook/tests/integration/object/cosi"
"github.com/rook/rook/tests/integration/object/notification"
@@ -214,6 +215,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
sharedObjectStore := sharedstore.Create(s.T(), k8sh, installer, tlsEnable,
bucketowner.Namespace,
bucketquota.Namespace,
bucketrw.Namespace,
userkeys.Namespace,
topickafka.Namespace,
@@ -225,6 +227,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
defer sharedObjectStore.Destroy()
bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, sharedObjectStore)
bucketquota.TestObjectBucketClaimQuota(s.T(), k8sh, sharedObjectStore)
bucketrw.TestObjectBucketClaimReadWrite(s.T(), k8sh, sharedObjectStore)
userkeys.TestObjectStoreUserKeys(s.T(), k8sh, sharedObjectStore)
topickafka.TestBucketTopicKafka(s.T(), k8sh, sharedObjectStore)
@@ -290,220 +293,6 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
logger.Info("OBC, Secret and ConfigMap created")
})
t.Run("S3 access to OBC bucket", func(t *testing.T) {
var s3client *rgw.S3Agent
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
insecure := objectStore.Spec.IsTLSEnabled()
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
assert.Nil(t, err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
t.Run("user quota enforcement", func(t *testing.T) {
_, poErr := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey1, contentType)
assert.Nil(t, poErr)
_, poErr = s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey2, contentType)
assert.Nil(t, poErr)
logger.Infof("Testing the max object limit")
quotaEnforced := utils.Retry(30, 2*time.Second, "user quota enforced", func() bool {
_, err := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey3, contentType)
if err != nil {
return true
}
// delete so next attempt creates a new object rather than overwriting
s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey3) //nolint:errcheck
return false
})
assert.True(t, quotaEnforced)
})
t.Run("update user quota limits", func(t *testing.T) {
poErr := helper.BucketClient.UpdateObc(obcName, bucketStorageClassName, bucketname, newMaxObject, true)
assert.Nil(t, poErr)
updated := utils.Retry(20, 2*time.Second, "OBC is updated", func() bool {
return helper.BucketClient.CheckOBMaxObject(obcName, newMaxObject)
})
assert.True(t, updated)
logger.Infof("Testing the updated object limit")
_, poErr = s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey3, contentType)
assert.NoError(t, poErr)
quotaEnforced := utils.Retry(30, 2*time.Second, "updated user quota enforced", func() bool {
_, putErr := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey4, contentType)
if putErr != nil {
return true
}
// delete so next attempt creates a new object rather than overwriting
s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey4) //nolint:errcheck
return false
})
assert.True(t, quotaEnforced)
})
t.Run("delete objects", func(t *testing.T) {
_, delobjErr := s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey1)
assert.Nil(t, delobjErr)
_, delobjErr = s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey2)
assert.Nil(t, delobjErr)
_, delobjErr = s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey3)
assert.Nil(t, delobjErr)
logger.Info("Objects deleted on bucket successfully")
})
})
// this test deviates from the others in this package in that it does not
// rely on kubectl and uses the k8s api directly
t.Run("OBC bucket quota enforcement", func(t *testing.T) {
bucketName := "bucket-quota-test"
var obName string
var s3client *rgw.S3Agent
t.Run("create obc with bucketMaxObjects", func(t *testing.T) {
newObc := v1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: bucketName,
Namespace: namespace,
},
Spec: v1alpha1.ObjectBucketClaimSpec{
BucketName: bucketName,
StorageClassName: bucketStorageClassName,
AdditionalConfig: map[string]string{
"bucketMaxObjects": "1",
},
},
}
_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Create(ctx, &newObc, metav1.CreateOptions{})
assert.Nil(t, err)
obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool {
liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
if err != nil {
return false
}
return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound
})
assert.True(t, obcBound)
// wait until obc is Bound to lookup the ob name
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
obName = obc.Spec.ObjectBucketName
obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool {
liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
if err != nil {
return false
}
return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound
})
assert.True(t, obBound)
// verify that bucketMaxObjects is set
assert.True(t, obc.Spec.AdditionalConfig["bucketMaxObjects"] == "1")
// as the tests are running external to k8s, the internal svc can't be used
labelSelector := "rgw=" + storeName
services, err := k8sh.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
assert.Nil(t, err)
assert.Equal(t, 1, len(services.Items))
s3endpoint := services.Items[0].Spec.ClusterIP + ":80"
secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"])
s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"])
insecure := objectStore.Spec.IsTLSEnabled()
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
assert.Nil(t, err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
})
t.Run("bucketMaxObjects quota is enforced", func(t *testing.T) {
// first object should succeed
_, err := s3client.PutObjectInBucket(ctx, bucketName, ObjBody, ObjectKey1, contentType)
assert.Nil(t, err)
// second object should fail as bucket quota is 1
_, err = s3client.PutObjectInBucket(ctx, bucketName, ObjBody, ObjectKey2, contentType)
assert.Error(t, err)
// cleanup bucket
_, err = s3client.DeleteObjectInBucket(ctx, bucketName, ObjectKey1)
assert.Nil(t, err)
_, err = s3client.DeleteObjectInBucket(ctx, bucketName, ObjectKey2)
assert.Nil(t, err)
})
t.Run("change quota to bucketMaxSize", func(t *testing.T) {
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
obc.Spec.AdditionalConfig = map[string]string{"bucketMaxSize": "4Ki"}
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{})
assert.Nil(t, err)
obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool {
liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
if err != nil {
return false
}
return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound
})
assert.True(t, obcBound)
// wait until obc is Bound to lookup the ob name
obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
assert.Nil(t, err)
obName = obc.Spec.ObjectBucketName
obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool {
liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
if err != nil {
return false
}
return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound
})
assert.True(t, obBound)
// verify that bucketMaxSize is set
assert.True(t, obc.Spec.AdditionalConfig["bucketMaxSize"] == "4Ki")
})
t.Run("bucketMaxSize quota is enforced", func(t *testing.T) {
// first ~3KiB Object should succeed
_, err := s3client.PutObjectInBucket(ctx, bucketName, strings.Repeat("1", 3072), ObjectKey1, contentType)
assert.Nil(t, err)
// second ~2KiB Object should fail as bucket quota is 4KiB
_, err = s3client.PutObjectInBucket(ctx, bucketName, strings.Repeat("2", 2048), ObjectKey2, contentType)
assert.Error(t, err)
// cleanup bucket
_, err = s3client.DeleteObjectInBucket(ctx, bucketName, ObjectKey1)
assert.Nil(t, err)
_, err = s3client.DeleteObjectInBucket(ctx, bucketName, ObjectKey2)
assert.Nil(t, err)
})
t.Run("delete bucket", func(t *testing.T) {
err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Delete(ctx, bucketName, metav1.DeleteOptions{})
assert.Nil(t, err)
absent := utils.Retry(20, 2*time.Second, "OBC is absent", func() bool {
_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
return err != nil
})
assert.True(t, absent)
absent = utils.Retry(20, 2*time.Second, "OB is absent", func() bool {
_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
return err != nil
})
assert.True(t, absent)
})
})
t.Run("OBC bucket policy management", func(t *testing.T) {
bucketName := "bucket-policy-test"
var obName string
+2 -2
View File
@@ -33,6 +33,7 @@ must not collide with std-lib package names (no `io`, no `http`).
| test package | operator package | covers |
|---|---|---|
| `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling |
| `bucket/quota` | `object/bucket` | OBC maxObjects user quota + bucketMaxObjects/bucketMaxSize bucket quota |
| `bucket/rw` | `object/bucket` | OBC S3 read/write/delete + OBC-stays-Bound |
| `cosi` | `object/cosi` | CephCOSIDriver + COSI bucket provisioning |
| `notification` | `object/notification` | CephBucketNotification HTTP endpoint delivery |
@@ -40,7 +41,7 @@ must not collide with std-lib package names (no `io`, no `http`).
| `user/caps` | `object/user` | user capabilities |
| `user/keys` | `object/user` | explicit S3 key management |
| `user/opmask` | `object/user` | user op_mask |
| reserved: `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions |
| reserved: `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions |
Shared utilities live under `util/`:
@@ -206,7 +207,6 @@ verification; wire the dispatcher; delete the old code; retire
| old test | target package(s) | still needs (build in that PR) |
|---|---|---|
| `testObjectStoreOperations` user/bucket quotas | `bucket/quota` | — |
| `testObjectStoreOperations` bucket policy | `bucket/policy` | — |
| `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | move `lifecycleCmpOpts` out of the dispatcher |
| `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `wait4` |
@@ -0,0 +1,215 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package quota
import (
"context"
"fmt"
"strings"
"testing"
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
rgw "github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/tests/framework/utils"
"github.com/rook/rook/tests/integration/object/util/fixture"
"github.com/rook/rook/tests/integration/object/util/obc"
"github.com/rook/rook/tests/integration/object/util/sharedstore"
"github.com/rook/rook/tests/integration/object/util/wait4"
)
const (
body = "test bucket quota payload"
contentType = "text/plain"
key1 = "obj1"
key2 = "obj2"
key3 = "obj3"
key4 = "obj4"
)
// obAdditionalConfig returns a predicate matching an ObjectBucket whose endpoint
// additionalConfig has key set to value, the sign that the provisioner has applied
// a quota change to the backing bucket.
func obAdditionalConfig(key, value string) func(*bktv1alpha1.ObjectBucket) bool {
return func(ob *bktv1alpha1.ObjectBucket) bool {
if ob.Spec.Connection == nil || ob.Spec.Connection.Endpoint == nil {
return false
}
return ob.Spec.Connection.Endpoint.AdditionalConfigData[key] == value
}
}
// requireQuotaEnforced waits until putting key into bucket is rejected. rgw
// enforces user quota against per-instance cached stats, so enforcement can lag
// recent writes; an unexpectedly-successful put is deleted so the next attempt
// creates a new object rather than overwriting.
func requireQuotaEnforced(ctx context.Context, t *testing.T, s3 *rgw.S3Agent, bucket, key string) {
t.Helper()
wait4.RequireEventually(ctx, t, wait4.TimeoutMedium, fmt.Sprintf("quota rejects object %q", key), func(ctx context.Context) error {
_, err := s3.PutObjectInBucket(ctx, bucket, body, key, contentType)
if err == nil {
s3.DeleteObjectInBucket(ctx, bucket, key) //nolint:errcheck
return fmt.Errorf("quota not yet enforced: put %q succeeded", key)
}
return nil
})
}
const Namespace = "test-bucketquota"
func TestObjectBucketClaimQuota(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
var (
defaultName = Namespace
objectStore = store.ObjectStore()
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
}
storageClass = obc.StorageClass(defaultName, objectStore)
obc1 = &bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-obc1",
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName + "-obc1",
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{"maxObjects": "2"},
},
}
obc2 = &bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-obc2",
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName + "-obc2",
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{"bucketMaxObjects": "1"},
},
}
obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name)
obClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
)
t.Run("ObjectBucketClaim quota", func(t *testing.T) {
ctx := t.Context()
fixture.RequireNamespace(t, k8sh, ns)
fixture.RequireStorageClass(t, k8sh, storageClass)
liveObc1 := obc.RequireBound(ctx, t, k8sh, obc1)
s3a := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc1.Name)
t.Run(fmt.Sprintf("maxObjects quota is enforced on obc %q", obc1.Name), func(t *testing.T) {
_, err := s3a.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key1, contentType)
require.NoError(t, err)
_, err = s3a.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key2, contentType)
require.NoError(t, err)
requireQuotaEnforced(ctx, t, s3a, obc1.Spec.BucketName, key3)
})
t.Run(fmt.Sprintf("raise maxObjects on obc %q", obc1.Name), func(t *testing.T) {
obc.Update(ctx, t, k8sh, ns.Name, obc1.Name, func(live *bktv1alpha1.ObjectBucketClaim) {
live.Spec.AdditionalConfig["maxObjects"] = "3"
})
wait4.RequireCondition(ctx, t, obClient, liveObc1.Spec.ObjectBucketName,
obAdditionalConfig("maxObjects", "3"), wait4.TimeoutShort)
})
t.Run(fmt.Sprintf("raised maxObjects quota admits object %q", key3), func(t *testing.T) {
_, err := s3a.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key3, contentType)
require.NoError(t, err)
requireQuotaEnforced(ctx, t, s3a, obc1.Spec.BucketName, key4)
})
t.Run(fmt.Sprintf("delete objects in bucket %q", obc1.Spec.BucketName), func(t *testing.T) {
for _, key := range []string{key1, key2, key3} {
_, err := s3a.DeleteObjectInBucket(ctx, obc1.Spec.BucketName, key)
assert.NoError(t, err)
}
})
t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) {
obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc1.Name)
})
liveObc2 := obc.RequireBound(ctx, t, k8sh, obc2)
s3b := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc2.Name)
t.Run(fmt.Sprintf("bucketMaxObjects quota is enforced on obc %q", obc2.Name), func(t *testing.T) {
_, err := s3b.PutObjectInBucket(ctx, obc2.Spec.BucketName, body, key1, contentType)
require.NoError(t, err)
// unlike the rgw-cached user quota, bucket quota applies immediately
_, err = s3b.PutObjectInBucket(ctx, obc2.Spec.BucketName, body, key2, contentType)
assert.Error(t, err)
for _, key := range []string{key1, key2} {
_, err := s3b.DeleteObjectInBucket(ctx, obc2.Spec.BucketName, key)
assert.NoError(t, err)
}
})
t.Run(fmt.Sprintf("switch obc %q to a bucketMaxSize quota", obc2.Name), func(t *testing.T) {
obc.Update(ctx, t, k8sh, ns.Name, obc2.Name, func(live *bktv1alpha1.ObjectBucketClaim) {
live.Spec.AdditionalConfig = map[string]string{"bucketMaxSize": "4Ki"}
})
wait4.RequireCondition(ctx, t, obClient, liveObc2.Spec.ObjectBucketName,
obAdditionalConfig("bucketMaxSize", "4Ki"), wait4.TimeoutShort)
})
t.Run(fmt.Sprintf("bucketMaxSize quota is enforced on obc %q", obc2.Name), func(t *testing.T) {
_, err := s3b.PutObjectInBucket(ctx, obc2.Spec.BucketName, strings.Repeat("1", 3072), key1, contentType)
require.NoError(t, err)
_, err = s3b.PutObjectInBucket(ctx, obc2.Spec.BucketName, strings.Repeat("2", 2048), key2, contentType)
assert.Error(t, err)
for _, key := range []string{key1, key2} {
_, err := s3b.DeleteObjectInBucket(ctx, obc2.Spec.BucketName, key)
assert.NoError(t, err)
}
})
t.Run(fmt.Sprintf("delete obc %q", obc2.Name), func(t *testing.T) {
obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc2.Name)
})
t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) {
list, err := obcClient.List(ctx, metav1.ListOptions{})
require.NoError(t, err)
assert.Len(t, list.Items, 0)
})
})
}