Merge pull request #17879 from jhoblitt/convert-bucket-rw-to-sharedstore
test: convert the object bucket read/write suite to the shared store
This commit is contained in:
9 files changed
+288
-166
No files matched your search
@@ -45,6 +45,7 @@ import (
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner"
|
||||
bucketrw "github.com/rook/rook/tests/integration/object/bucket/rw"
|
||||
"github.com/rook/rook/tests/integration/object/cosi"
|
||||
"github.com/rook/rook/tests/integration/object/notification"
|
||||
topickafka "github.com/rook/rook/tests/integration/object/topic/kafka"
|
||||
@@ -213,6 +214,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
|
||||
|
||||
sharedObjectStore := sharedstore.Create(s.T(), k8sh, installer, tlsEnable,
|
||||
bucketowner.Namespace,
|
||||
bucketrw.Namespace,
|
||||
userkeys.Namespace,
|
||||
topickafka.Namespace,
|
||||
useropmask.Namespace,
|
||||
@@ -223,6 +225,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
|
||||
defer sharedObjectStore.Destroy()
|
||||
|
||||
bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, sharedObjectStore)
|
||||
bucketrw.TestObjectBucketClaimReadWrite(s.T(), k8sh, sharedObjectStore)
|
||||
userkeys.TestObjectStoreUserKeys(s.T(), k8sh, sharedObjectStore)
|
||||
topickafka.TestBucketTopicKafka(s.T(), k8sh, sharedObjectStore)
|
||||
useropmask.TestObjectStoreUserOpMask(s.T(), k8sh, sharedObjectStore)
|
||||
@@ -298,19 +301,10 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
|
||||
assert.Nil(t, err)
|
||||
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
|
||||
|
||||
t.Run("put object", func(t *testing.T) {
|
||||
t.Run("user quota enforcement", func(t *testing.T) {
|
||||
_, poErr := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey1, contentType)
|
||||
assert.Nil(t, poErr)
|
||||
})
|
||||
|
||||
t.Run("get object", func(t *testing.T) {
|
||||
read, err := s3client.GetObjectInBucket(ctx, bucketname, ObjectKey1)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, ObjBody, read)
|
||||
})
|
||||
|
||||
t.Run("user quota enforcement", func(t *testing.T) {
|
||||
_, poErr := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey2, contentType)
|
||||
_, poErr = s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey2, contentType)
|
||||
assert.Nil(t, poErr)
|
||||
logger.Infof("Testing the max object limit")
|
||||
quotaEnforced := utils.Retry(30, 2*time.Second, "user quota enforced", func() bool {
|
||||
@@ -1007,18 +1001,6 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("Regression check: OBC does not revert to Pending phase", func(t *testing.T) {
|
||||
// A bug exists in older versions of lib-bucket-provisioner that will revert a bucket and claim
|
||||
// back to "Pending" phase after being created and initially "Bound" by looping infinitely in
|
||||
// the bucket provision/creation loop. Verify that the OBC is "Bound" and stays that way.
|
||||
// The OBC reconcile loop runs again immediately b/c the OBC is modified to refer to its OB.
|
||||
// Wait a short amount of time before checking just to be safe.
|
||||
created := utils.Retry(15, 2*time.Second, "OBC is created", func() bool {
|
||||
return helper.BucketClient.CheckOBC(obcName, "bound")
|
||||
})
|
||||
assert.True(t, created)
|
||||
})
|
||||
|
||||
t.Run("delete CephObjectStore should be blocked by OBC bucket and CephObjectStoreUser", func(t *testing.T) {
|
||||
deleteObjectStore(t, k8sh, namespace, storeName)
|
||||
|
||||
|
||||
@@ -33,31 +33,31 @@ must not collide with std-lib package names (no `io`, no `http`).
|
||||
| test package | operator package | covers |
|
||||
|---|---|---|
|
||||
| `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling |
|
||||
| `bucket/rw` | `object/bucket` | OBC S3 read/write/delete + OBC-stays-Bound |
|
||||
| `cosi` | `object/cosi` | CephCOSIDriver + COSI bucket provisioning |
|
||||
| `notification` | `object/notification` | CephBucketNotification HTTP endpoint delivery |
|
||||
| `topic/kafka` | `object/topic` | CephBucketTopic kafka endpoints |
|
||||
| `user/caps` | `object/user` | user capabilities |
|
||||
| `user/keys` | `object/user` | explicit S3 key management |
|
||||
| `user/opmask` | `object/user` | user op_mask |
|
||||
| reserved: `bucket/rw`, `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions |
|
||||
| reserved: `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions |
|
||||
|
||||
Shared utilities live under `util/`:
|
||||
|
||||
- `wait4` — waits for object-test state: watch-based `Assert/Require` ×
|
||||
`Create/Delete/Condition/Absent` for k8s resources, `Assert/RequireEventually`
|
||||
polling for non-k8s state, `Assert/RequirePodLog` for a matching line in a
|
||||
pod's log stream, and the shared timeout tiers.
|
||||
- `ready` — readiness predicates for the wait4 helpers (`ObjectStoreUser`,
|
||||
`BucketTopic`, `OBCBound`, ...).
|
||||
pod's log stream, the readiness predicates (`ObjectStoreUser`, `BucketTopic`,
|
||||
`OBCBound`, ...), and the shared timeout tiers.
|
||||
- `fixture` — create-with-`t.Cleanup` helpers for pure-cleanup resources
|
||||
(namespaces, StorageClasses).
|
||||
- `obc` — pure constructors for the resources ObjectBucketClaim tests build
|
||||
(the provisioner `StorageClass`; OBCs and per-OBC S3 clients as `bucket/*`
|
||||
is converted).
|
||||
- `obc` — ObjectBucketClaim helpers: the provisioner `StorageClass`
|
||||
constructor, the create/bound and delete/absent lifecycle waiters, and a
|
||||
per-OBC S3 client.
|
||||
- `secrets` — verification helpers for the Secret references object CRDs
|
||||
publish in their status, shared by more than one package.
|
||||
- `sharedstore` — the shared CephObjectStore fixture.
|
||||
- `admin`, `sns`, `s3`, `tls` — client builders and TLS cert generation.
|
||||
- `client` — rgw admin, SNS, and S3 client builders and TLS cert generation.
|
||||
|
||||
## Anatomy of a package
|
||||
|
||||
@@ -83,7 +83,7 @@ func TestObjectStoreUserCaps(t *testing.T, k8sh *utils.K8sHelper, store *shareds
|
||||
fixture.RequireNamespace(t, k8sh, ns)
|
||||
|
||||
t.Run(fmt.Sprintf("create CephObjectStoreUser %q", osu1.Name), func(t *testing.T) {
|
||||
wait4.RequireCreate(ctx, t, osuClient, &osu1, ready.ObjectStoreUser, wait4.TimeoutLong)
|
||||
wait4.RequireCreate(ctx, t, osuClient, &osu1, wait4.ObjectStoreUser, wait4.TimeoutLong)
|
||||
})
|
||||
|
||||
// ... ordered subtests: act, verify, act, verify ...
|
||||
@@ -121,7 +121,7 @@ Rules encoded in that shape:
|
||||
`RequireCreate` (create + wait ready), `AssertDelete`/`RequireDelete`
|
||||
(delete + wait gone), `RequireCondition`/`AssertCondition` (wait for a
|
||||
predicate on an existing resource), `AssertAbsent`/`RequireAbsent` (wait
|
||||
for cascade deletion you did not issue). Predicates come from `util/ready`
|
||||
for cascade deletion you did not issue). Predicates come from `wait4`
|
||||
or are inlined when test-specific.
|
||||
- State NOT visible to the Kubernetes API (rgw admin ops, S3, SNS) uses
|
||||
`wait4.AssertEventually`/`RequireEventually`, the assert/require-flavored
|
||||
@@ -206,10 +206,9 @@ verification; wire the dispatcher; delete the old code; retire
|
||||
|
||||
| old test | target package(s) | still needs (build in that PR) |
|
||||
|---|---|---|
|
||||
| `testObjectStoreOperations` S3 I/O + OBC-stays-Bound | `bucket/rw` | per-OBC S3 client (TLS-aware, from OBC secret) |
|
||||
| `testObjectStoreOperations` user/bucket quotas | `bucket/quota` | per-OBC S3 client |
|
||||
| `testObjectStoreOperations` bucket policy | `bucket/policy` | per-OBC S3 client |
|
||||
| `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | per-OBC S3 client, move `lifecycleCmpOpts` out of the dispatcher |
|
||||
| `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `ready` |
|
||||
| `testObjectStoreOperations` user/bucket quotas | `bucket/quota` | — |
|
||||
| `testObjectStoreOperations` bucket policy | `bucket/policy` | — |
|
||||
| `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | move `lifecycleCmpOpts` out of the dispatcher |
|
||||
| `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `wait4` |
|
||||
| `createCephObjectStore`/`runObjectE2ETestLite`/deletion asserts + zone.json canary | `tests/integration/object/lifecycle` | store create/health/delete helpers, `Sharedstore.Installer()` accessor |
|
||||
| upgrade-suite object usage | stays in upgrade suite | switch to typed clients + `wait4`/`ready` |
|
||||
| upgrade-suite object usage | stays in upgrade suite | switch to typed clients + `wait4` |
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/rook/rook/tests/integration/object/util/fixture"
|
||||
"github.com/rook/rook/tests/integration/object/util/obc"
|
||||
"github.com/rook/rook/tests/integration/object/util/sharedstore"
|
||||
"github.com/rook/rook/tests/integration/object/util/wait4"
|
||||
)
|
||||
@@ -52,7 +53,7 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, store
|
||||
},
|
||||
}
|
||||
|
||||
storageClass = fixture.StorageClass(defaultName, objectStore)
|
||||
storageClass = obc.StorageClass(defaultName, objectStore)
|
||||
|
||||
// test user without any quotas set
|
||||
osu1 = cephv1.CephObjectStoreUser{
|
||||
@@ -264,8 +265,8 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, store
|
||||
t.Run(fmt.Sprintf("obc %q has no bucketOwner", obc1.Name), func(t *testing.T) {
|
||||
// verify that bucketOwner is unset on the live obc
|
||||
wait4.AssertCondition(ctx, t, obcClient, obc1.Name,
|
||||
func(obc *bktv1alpha1.ObjectBucketClaim) bool {
|
||||
_, ok := obc.Spec.AdditionalConfig["bucketOwner"]
|
||||
func(liveObc *bktv1alpha1.ObjectBucketClaim) bool {
|
||||
_, ok := liveObc.Spec.AdditionalConfig["bucketOwner"]
|
||||
return !ok
|
||||
},
|
||||
wait4.TimeoutShort)
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
Copyright 2026 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rw
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/rook/rook/tests/integration/object/util/fixture"
|
||||
"github.com/rook/rook/tests/integration/object/util/obc"
|
||||
"github.com/rook/rook/tests/integration/object/util/sharedstore"
|
||||
"github.com/rook/rook/tests/integration/object/util/wait4"
|
||||
)
|
||||
|
||||
const Namespace = "test-bucketrw"
|
||||
|
||||
func TestObjectBucketClaimReadWrite(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
|
||||
var (
|
||||
defaultName = Namespace
|
||||
objectStore = store.ObjectStore()
|
||||
|
||||
ns = &corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: defaultName,
|
||||
},
|
||||
}
|
||||
|
||||
storageClass = obc.StorageClass(defaultName, objectStore)
|
||||
|
||||
obc1 = &bktv1alpha1.ObjectBucketClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: defaultName + "-obc1",
|
||||
Namespace: ns.Name,
|
||||
},
|
||||
Spec: bktv1alpha1.ObjectBucketClaimSpec{
|
||||
BucketName: defaultName + "-obc1",
|
||||
StorageClassName: storageClass.Name,
|
||||
},
|
||||
}
|
||||
|
||||
body = "test bucket read/write payload"
|
||||
contentType = "text/plain"
|
||||
key = "obj1"
|
||||
|
||||
obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name)
|
||||
)
|
||||
|
||||
t.Run("ObjectBucketClaim read/write", func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
|
||||
fixture.RequireNamespace(t, k8sh, ns)
|
||||
fixture.RequireStorageClass(t, k8sh, storageClass)
|
||||
|
||||
obc.RequireBound(ctx, t, k8sh, obc1)
|
||||
s3 := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc1.Name)
|
||||
|
||||
t.Run(fmt.Sprintf("put object %q", key), func(t *testing.T) {
|
||||
_, err := s3.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key, contentType)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("get object %q round-trips", key), func(t *testing.T) {
|
||||
read, err := s3.GetObjectInBucket(ctx, obc1.Spec.BucketName, key)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, body, read)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete object %q", key), func(t *testing.T) {
|
||||
_, err := s3.DeleteObjectInBucket(ctx, obc1.Spec.BucketName, key)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("obc %q stays Bound", obc1.Name), func(t *testing.T) {
|
||||
// regression: older lib-bucket-provisioner versions looped in the
|
||||
// provision loop and reverted a Bound OBC back to Pending
|
||||
wait4.AssertCondition(ctx, t, obcClient, obc1.Name, wait4.OBCBound, wait4.TimeoutShort)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) {
|
||||
obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc1.Name)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) {
|
||||
list, err := obcClient.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, list.Items, 0)
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -37,8 +37,8 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rgw "github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/rook/rook/tests/integration/object/util/client"
|
||||
"github.com/rook/rook/tests/integration/object/util/fixture"
|
||||
"github.com/rook/rook/tests/integration/object/util/obc"
|
||||
"github.com/rook/rook/tests/integration/object/util/sharedstore"
|
||||
"github.com/rook/rook/tests/integration/object/util/wait4"
|
||||
)
|
||||
@@ -110,11 +110,11 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
sinkSel = labels.SelectorFromSet(labels.Set{"app": sinkName})
|
||||
sinkURI = fmt.Sprintf("http://%s.%s:%d", sinkName, ns.Name, sinkPort)
|
||||
|
||||
storageClass = fixture.StorageClass(defaultName, objectStore)
|
||||
storageClass = obc.StorageClass(defaultName, objectStore)
|
||||
|
||||
topic = httpTopic(defaultName+"-topic", ns.Name, objectStore, sinkURI)
|
||||
notification = bucketNotification(defaultName+"-notification", ns.Name, topic.Name)
|
||||
obc = notificationOBC(defaultName+"-obc", ns.Name, storageClass.Name, notification.Name)
|
||||
obc1 = notificationOBC(defaultName+"-obc", ns.Name, storageClass.Name, notification.Name)
|
||||
|
||||
body = "test bucket notification payload"
|
||||
contentType = "text/plain"
|
||||
@@ -145,14 +145,15 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
s3agent := requireBoundOBC(ctx, t, k8sh, store, obc)
|
||||
obc.RequireBound(ctx, t, k8sh, obc1)
|
||||
s3agent := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obc1.Name)
|
||||
|
||||
t.Run(fmt.Sprintf("notification %q is configured on bucket %q", notification.Name, obc.Spec.BucketName), func(t *testing.T) {
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true)
|
||||
t.Run(fmt.Sprintf("notification %q is configured on bucket %q", notification.Name, obc1.Spec.BucketName), func(t *testing.T) {
|
||||
requireBucketNotification(ctx, t, s3agent, obc1.Spec.BucketName, notification.Name, true)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("put object %q delivers a notification", key1), func(t *testing.T) {
|
||||
_, err := s3agent.PutObjectInBucket(ctx, obc.Spec.BucketName, body, key1, contentType)
|
||||
_, err := s3agent.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key1, contentType)
|
||||
require.NoError(t, err)
|
||||
|
||||
requireNotificationReceived(ctx, t, k8sh, ns.Name, sinkSel, putEvent, key1)
|
||||
@@ -160,7 +161,7 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete object %q delivers a notification", key1), func(t *testing.T) {
|
||||
_, err := s3agent.DeleteObjectInBucket(ctx, obc.Spec.BucketName, key1)
|
||||
_, err := s3agent.DeleteObjectInBucket(ctx, obc1.Spec.BucketName, key1)
|
||||
require.NoError(t, err)
|
||||
|
||||
requireNotificationReceived(ctx, t, k8sh, ns.Name, sinkSel, deleteEvent, key1)
|
||||
@@ -168,43 +169,43 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
})
|
||||
|
||||
t.Run("an unrelated OBC label change leaves the notification configured", func(t *testing.T) {
|
||||
t.Run(fmt.Sprintf("add a non-notification label to obc %q", obc.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) {
|
||||
t.Run(fmt.Sprintf("add a non-notification label to obc %q", obc1.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc1.Name, func(l map[string]string) {
|
||||
l["test-label"] = "test-value"
|
||||
})
|
||||
})
|
||||
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true)
|
||||
requireBucketNotification(ctx, t, s3agent, obc1.Spec.BucketName, notification.Name, true)
|
||||
|
||||
t.Run(fmt.Sprintf("remove the non-notification label from obc %q", obc.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) {
|
||||
t.Run(fmt.Sprintf("remove the non-notification label from obc %q", obc1.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc1.Name, func(l map[string]string) {
|
||||
delete(l, "test-label")
|
||||
})
|
||||
})
|
||||
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true)
|
||||
requireBucketNotification(ctx, t, s3agent, obc1.Spec.BucketName, notification.Name, true)
|
||||
})
|
||||
|
||||
t.Run("removing the notification label removes the notification from the bucket", func(t *testing.T) {
|
||||
t.Run(fmt.Sprintf("remove the notification label from obc %q", obc.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) {
|
||||
t.Run(fmt.Sprintf("remove the notification label from obc %q", obc1.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc1.Name, func(l map[string]string) {
|
||||
delete(l, notificationLabel(notification.Name))
|
||||
})
|
||||
})
|
||||
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, false)
|
||||
requireBucketNotification(ctx, t, s3agent, obc1.Spec.BucketName, notification.Name, false)
|
||||
})
|
||||
|
||||
t.Run("an unsubscribed bucket delivers no notifications", func(t *testing.T) {
|
||||
t.Run(fmt.Sprintf("put object %q", key2), func(t *testing.T) {
|
||||
_, err := s3agent.PutObjectInBucket(ctx, obc.Spec.BucketName, body, key2, contentType)
|
||||
_, err := s3agent.PutObjectInBucket(ctx, obc1.Spec.BucketName, body, key2, contentType)
|
||||
require.NoError(t, err)
|
||||
|
||||
assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, putEvent, key2)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete object %q", key2), func(t *testing.T) {
|
||||
_, err := s3agent.DeleteObjectInBucket(ctx, obc.Spec.BucketName, key2)
|
||||
_, err := s3agent.DeleteObjectInBucket(ctx, obc1.Spec.BucketName, key2)
|
||||
require.NoError(t, err)
|
||||
|
||||
assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, deleteEvent, key2)
|
||||
@@ -224,13 +225,13 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("add the notification label for %q to obc %q", addNotification.Name, obc.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) {
|
||||
t.Run(fmt.Sprintf("add the notification label for %q to obc %q", addNotification.Name, obc1.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc1.Name, func(l map[string]string) {
|
||||
l[notificationLabel(addNotification.Name)] = addNotification.Name
|
||||
})
|
||||
})
|
||||
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, addNotification.Name, true)
|
||||
requireBucketNotification(ctx, t, s3agent, obc1.Spec.BucketName, addNotification.Name, true)
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketNotification %q", addNotification.Name), func(t *testing.T) {
|
||||
wait4.AssertDelete(ctx, t, bnClient, addNotification.Name, wait4.TimeoutShort)
|
||||
@@ -244,9 +245,10 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
t.Run("a notification created before its topic is configured once the topic exists", func(t *testing.T) {
|
||||
revTopic := httpTopic(defaultName+"-topic-rev", ns.Name, objectStore, sinkURI)
|
||||
revNotification := bucketNotification(defaultName+"-notification-rev", ns.Name, revTopic.Name)
|
||||
revOBC := notificationOBC(defaultName+"-obc-rev", ns.Name, storageClass.Name, revNotification.Name)
|
||||
obcRev := notificationOBC(defaultName+"-obc-rev", ns.Name, storageClass.Name, revNotification.Name)
|
||||
|
||||
revS3 := requireBoundOBC(ctx, t, k8sh, store, revOBC)
|
||||
obc.RequireBound(ctx, t, k8sh, obcRev)
|
||||
revS3 := obc.NewS3Agent(ctx, t, k8sh, store, ns.Name, obcRev.Name)
|
||||
|
||||
t.Run(fmt.Sprintf("create CephBucketNotification %q before its topic", revNotification.Name), func(t *testing.T) {
|
||||
_, err := bnClient.Create(ctx, revNotification, metav1.CreateOptions{})
|
||||
@@ -257,8 +259,8 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
wait4.RequireCreate(ctx, t, btClient, revTopic, wait4.BucketTopic, wait4.TimeoutMedium)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("notification %q is configured on bucket %q once its topic exists", revNotification.Name, revOBC.Spec.BucketName), func(t *testing.T) {
|
||||
requireBucketNotification(ctx, t, revS3, revOBC.Spec.BucketName, revNotification.Name, true)
|
||||
t.Run(fmt.Sprintf("notification %q is configured on bucket %q once its topic exists", revNotification.Name, obcRev.Spec.BucketName), func(t *testing.T) {
|
||||
requireBucketNotification(ctx, t, revS3, obcRev.Spec.BucketName, revNotification.Name, true)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketNotification %q", revNotification.Name), func(t *testing.T) {
|
||||
@@ -269,13 +271,13 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
wait4.AssertDelete(ctx, t, btClient, revTopic.Name, wait4.TimeoutShort)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete obc %q", revOBC.Name), func(t *testing.T) {
|
||||
deleteOBCAndBucket(ctx, t, k8sh, ns.Name, revOBC.Name)
|
||||
t.Run(fmt.Sprintf("delete obc %q", obcRev.Name), func(t *testing.T) {
|
||||
obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obcRev.Name)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete obc %q", obc.Name), func(t *testing.T) {
|
||||
deleteOBCAndBucket(ctx, t, k8sh, ns.Name, obc.Name)
|
||||
t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) {
|
||||
obc.DeleteAndWait(ctx, t, k8sh, ns.Name, obc1.Name)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketNotification %q", notification.Name), func(t *testing.T) {
|
||||
@@ -306,40 +308,6 @@ func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedst
|
||||
})
|
||||
}
|
||||
|
||||
// requireBoundOBC creates obc, waits for it and its backing ObjectBucket to bind,
|
||||
// and returns an S3 client for the provisioned bucket.
|
||||
func requireBoundOBC(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore, obc *bktv1alpha1.ObjectBucketClaim) *rgw.S3Agent {
|
||||
t.Helper()
|
||||
|
||||
obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc.Namespace)
|
||||
obClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
|
||||
|
||||
var agent *rgw.S3Agent
|
||||
t.Run(fmt.Sprintf("create obc %q", obc.Name), func(t *testing.T) {
|
||||
liveOBC := wait4.RequireCreate(ctx, t, obcClient, obc, wait4.OBCBound, wait4.TimeoutLong)
|
||||
wait4.RequireCondition(ctx, t, obClient, liveOBC.Spec.ObjectBucketName, wait4.OBBound, wait4.TimeoutShort)
|
||||
|
||||
agent = obcS3Agent(ctx, t, k8sh, store, obc.Namespace, obc.Name)
|
||||
})
|
||||
return agent
|
||||
}
|
||||
|
||||
// deleteOBCAndBucket deletes the OBC and waits for its backing ObjectBucket (and
|
||||
// thus the rgw bucket) to be garbage-collected by the provisioner.
|
||||
func deleteOBCAndBucket(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string) {
|
||||
t.Helper()
|
||||
|
||||
obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace)
|
||||
obClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
|
||||
|
||||
liveOBC, err := obcClient.Get(ctx, name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
obName := liveOBC.Spec.ObjectBucketName
|
||||
|
||||
wait4.AssertDelete(ctx, t, obcClient, name, wait4.TimeoutShort)
|
||||
wait4.AssertAbsent(ctx, t, obClient, obName, wait4.TimeoutShort)
|
||||
}
|
||||
|
||||
// updateOBCLabels applies mutate to a live OBC's labels and updates it.
|
||||
func updateOBCLabels(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string, mutate func(map[string]string)) {
|
||||
t.Helper()
|
||||
@@ -402,26 +370,6 @@ func requireHTTPSink(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, n
|
||||
})
|
||||
}
|
||||
|
||||
// obcS3Agent builds an S3 client for the bucket provisioned by obcName, using the
|
||||
// credentials in the OBC's secret and the shared store's endpoint.
|
||||
func obcS3Agent(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore, namespace, obcName string) *rgw.S3Agent {
|
||||
t.Helper()
|
||||
|
||||
secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, obcName, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
endpoint, err := client.GetS3Endpoint(store.ObjectStore(), k8sh, store.TLSEnabled())
|
||||
require.NoError(t, err)
|
||||
|
||||
agent, err := rgw.NewS3Agent(
|
||||
string(secret.Data["AWS_ACCESS_KEY_ID"]),
|
||||
string(secret.Data["AWS_SECRET_ACCESS_KEY"]),
|
||||
endpoint, true, nil, store.TLSEnabled(), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
return agent
|
||||
}
|
||||
|
||||
// requireBucketNotification waits until notificationName is (want=true) or is not
|
||||
// (want=false) configured on the rgw bucket, read via the S3 API.
|
||||
func requireBucketNotification(ctx context.Context, t *testing.T, agent *rgw.S3Agent, bucket, notificationName string, want bool) {
|
||||
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/rook/rook/tests/integration/object/util/fixture"
|
||||
"github.com/rook/rook/tests/integration/object/util/obc"
|
||||
"github.com/rook/rook/tests/integration/object/util/secrets"
|
||||
"github.com/rook/rook/tests/integration/object/util/sharedstore"
|
||||
"github.com/rook/rook/tests/integration/object/util/wait4"
|
||||
@@ -137,7 +138,7 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, store *sharedstor
|
||||
secret3 = kvSecret(ns.Name, defaultName+"-secret3", "user-name", "kafka-user3")
|
||||
secret4 = kvSecret(ns.Name, defaultName+"-secret4", "password", "kafka-pass4")
|
||||
|
||||
storageClass = fixture.StorageClass(defaultName, objectStore)
|
||||
storageClass = obc.StorageClass(defaultName, objectStore)
|
||||
|
||||
bt1 = &cephv1.CephBucketTopic{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rgw "github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
)
|
||||
@@ -73,3 +74,18 @@ func GetS3Endpoint(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, t
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
// NewS3Agent builds an S3 client for objectStore using the given credentials.
|
||||
func NewS3Agent(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, tlsEnable bool, accessKey, secretKey string) (*rgw.S3Agent, error) {
|
||||
endpoint, err := GetS3Endpoint(objectStore, k8sh, tlsEnable)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get s3 endpoint")
|
||||
}
|
||||
|
||||
agent, err := rgw.NewS3Agent(accessKey, secretKey, endpoint, true, nil, tlsEnable, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create s3 agent")
|
||||
}
|
||||
|
||||
return agent, nil
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
/*
|
||||
Copyright 2026 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package fixture
|
||||
|
||||
import (
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
)
|
||||
|
||||
// StorageClass returns a StorageClass for the lib-bucket-provisioner bucket
|
||||
// provisioner backed by objectStore, for use by ObjectBucketClaims.
|
||||
func StorageClass(name string, objectStore *cephv1.CephObjectStore) *storagev1.StorageClass {
|
||||
return &storagev1.StorageClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
|
||||
Parameters: map[string]string{
|
||||
"objectStoreName": objectStore.Name,
|
||||
"objectStoreNamespace": objectStore.Namespace,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
Copyright 2026 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package obc holds the ObjectBucketClaim test helpers: the provisioner
|
||||
// StorageClass constructor, the bound/delete lifecycle waiters, and a per-OBC S3
|
||||
// client.
|
||||
package obc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
|
||||
"github.com/stretchr/testify/require"
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rgw "github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/rook/rook/tests/integration/object/util/client"
|
||||
"github.com/rook/rook/tests/integration/object/util/sharedstore"
|
||||
"github.com/rook/rook/tests/integration/object/util/wait4"
|
||||
)
|
||||
|
||||
// StorageClass returns a StorageClass for the lib-bucket-provisioner bucket
|
||||
// provisioner backed by objectStore, for use by ObjectBucketClaims.
|
||||
func StorageClass(name string, objectStore *cephv1.CephObjectStore) *storagev1.StorageClass {
|
||||
return &storagev1.StorageClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
|
||||
Parameters: map[string]string{
|
||||
"objectStoreName": objectStore.Name,
|
||||
"objectStoreNamespace": objectStore.Namespace,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// RequireBound creates obc and waits for it and its backing ObjectBucket to
|
||||
// bind, returning the live OBC; it aborts the caller if binding fails.
|
||||
func RequireBound(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, obc *bktv1alpha1.ObjectBucketClaim) *bktv1alpha1.ObjectBucketClaim {
|
||||
t.Helper()
|
||||
|
||||
obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc.Namespace)
|
||||
obClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
|
||||
|
||||
var live *bktv1alpha1.ObjectBucketClaim
|
||||
if !t.Run(fmt.Sprintf("create obc %q", obc.Name), func(t *testing.T) {
|
||||
live = wait4.RequireCreate(ctx, t, obcClient, obc, wait4.OBCBound, wait4.TimeoutLong)
|
||||
wait4.RequireCondition(ctx, t, obClient, live.Spec.ObjectBucketName, wait4.OBBound, wait4.TimeoutShort)
|
||||
}) {
|
||||
t.FailNow()
|
||||
}
|
||||
return live
|
||||
}
|
||||
|
||||
// NewS3Agent builds an S3 client for the bucket provisioned by obcName, using the
|
||||
// credentials in the OBC's secret and the store's endpoint.
|
||||
func NewS3Agent(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore, namespace, obcName string) *rgw.S3Agent {
|
||||
t.Helper()
|
||||
|
||||
secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, obcName, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
agent, err := client.NewS3Agent(
|
||||
store.ObjectStore(), k8sh, store.TLSEnabled(),
|
||||
string(secret.Data["AWS_ACCESS_KEY_ID"]),
|
||||
string(secret.Data["AWS_SECRET_ACCESS_KEY"]))
|
||||
require.NoError(t, err)
|
||||
|
||||
return agent
|
||||
}
|
||||
|
||||
// DeleteAndWait deletes the OBC and waits for its backing ObjectBucket (and thus
|
||||
// the rgw bucket) to be garbage-collected by the provisioner.
|
||||
func DeleteAndWait(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string) {
|
||||
t.Helper()
|
||||
|
||||
obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace)
|
||||
obClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
|
||||
|
||||
live, err := obcClient.Get(ctx, name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
obName := live.Spec.ObjectBucketName
|
||||
|
||||
wait4.AssertDelete(ctx, t, obcClient, name, wait4.TimeoutShort)
|
||||
wait4.AssertAbsent(ctx, t, obClient, obName, wait4.TimeoutShort)
|
||||
}
|
||||
Reference in new issue
Block a user