diff --git a/tests/integration/ceph_bucket_notification_test.go b/tests/integration/ceph_bucket_notification_test.go deleted file mode 100644 index 031661865..000000000 --- a/tests/integration/ceph_bucket_notification_test.go +++ /dev/null @@ -1,443 +0,0 @@ -/* -Copyright 2021 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package integration - -import ( - "context" - "testing" - "time" - - "github.com/rook/rook/pkg/daemon/ceph/client" - rgw "github.com/rook/rook/pkg/operator/ceph/object" - "github.com/rook/rook/tests/framework/clients" - "github.com/rook/rook/tests/framework/utils" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - httpServerName = "sample-http-server" - httpServerNameSpace = "default" - httpServerPort = "8080" - httpEndpointService = "http://" + httpServerName + "." + httpServerNameSpace + ":" + httpServerPort - putEvent = "ObjectCreated:Put" - deleteEvent = "ObjectRemoved:Delete" -) - -func testBucketNotifications(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) { - if utils.IsPlatformOpenShift() { - s.T().Skip("bucket notification tests skipped on openshift") - } - - bucketNotificationLabelPrefix := "bucket-notification-" - obcNamespace := "default" - - ctx := context.TODO() - clusterInfo := client.AdminTestClusterInfo(namespace) - t := s.T() - - context := k8sh.MakeContext() - objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{}) - assert.Nil(t, err) - rgwcontext, err := rgw.NewMultisiteContext(context, clusterInfo, objectStore) - assert.Nil(t, err) - - notificationName := "my-notification" - topicName := "my-topic" - appLabel := "app=" + httpServerName - - logger.Infof("Testing Bucket Notifications on %s", storeName) - - t.Run("create HTTP Endpoint for receiving notifications", func(t *testing.T) { - err := helper.TopicClient.CreateHTTPServer(httpServerName, httpServerNameSpace, httpServerPort) - assert.Nil(t, err) - }) - - t.Run("create CephBucketTopic", func(t *testing.T) { - err := helper.TopicClient.CreateTopic(topicName, storeName, httpEndpointService) - assert.Nil(t, err) - created := utils.Retry(30, 5*time.Second, "topic is created", func() bool { - return helper.TopicClient.CheckTopic(topicName) - }) - assert.True(t, created) - logger.Info("CephBucketTopic created successfully") - }) - - t.Run("create CephBucketNotification", func(t *testing.T) { - err := helper.NotificationClient.CreateNotification(notificationName, topicName) - assert.Nil(t, err) - created := utils.Retry(12, 2*time.Second, "notification is created", func() bool { - return helper.NotificationClient.CheckNotificationCR(notificationName) - }) - assert.True(t, created) - logger.Info("CephBucketNotification created successfully") - }) - - t.Run("create ObjectBucketClaim", func(t *testing.T) { - logger.Infof("create OBC %q with storageclass %q and notification %q", obcName, bucketStorageClassName, notificationName) - cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete") - assert.Nil(t, cobErr) - cobcErr := helper.BucketClient.CreateObcNotification(obcName, bucketStorageClassName, bucketname, notificationName, true) - assert.Nil(t, cobcErr) - - created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool { - return helper.BucketClient.CheckOBC(obcName, "bound") - }) - assert.True(t, created) - logger.Info("OBC created successfully") - - var bkt rgw.ObjectBucket - i := 0 - for i = 0; i < 4; i++ { - b, code, err := rgw.GetBucket(rgwcontext, bucketname) - if b != nil && err == nil { - bkt = *b - break - } - logger.Warningf("cannot get bucket %q, retrying... bucket: %v. code: %d, err: %v", bucketname, b, code, err) - logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i) - time.Sleep(5 * time.Second) - } - assert.NotEqual(t, 4, i) - assert.Equal(t, bucketname, bkt.Name) - logger.Info("OBC, Secret and ConfigMap created") - }) - - t.Run("perform s3 operations and check for notifications", func(t *testing.T) { - var s3client *rgw.S3Agent - s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName) - s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName) - s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName) - insecure := objectStore.Spec.IsTLSEnabled() - s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil) - - assert.Nil(t, err) - logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) - - t.Run("put object", func(t *testing.T) { - _, err := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey1, contentType) - assert.Nil(t, err) - }) - - t.Run("check for put bucket notification", func(t *testing.T) { - notificationReceived, err := helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, putEvent, ObjectKey1) - assert.True(t, notificationReceived) - assert.Nil(t, err) - // negative test case to confirm didn't receive any delete event notification - notificationReceived, err = helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, deleteEvent, ObjectKey1) - assert.False(t, notificationReceived) - assert.Nil(t, err) - }) - - t.Run("delete objects", func(t *testing.T) { - _, err := s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey1) - assert.Nil(t, err) - }) - - t.Run("check for delete bucket notification", func(t *testing.T) { - notificationReceived, err := helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, deleteEvent, ObjectKey1) - assert.True(t, notificationReceived) - assert.Nil(t, err) - // negative test case to confirm didn't receive any put event notification - notificationReceived, err = helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, putEvent, ObjectKey1) - assert.False(t, notificationReceived) - assert.Nil(t, err) - }) - }) - - t.Run("check CephBucketNotification created for bucket", func(t *testing.T) { - notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { - return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled()) - }) - assert.True(t, notificationPresent) - logger.Info("CephBucketNotification created successfully on bucket") - }) - - t.Run("add non-notification label to OBC", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{}) - assert.Nil(t, err) - obc.Labels["test-label"] = "test-value" - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - // check whether existing bucket notification unaffected - notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { - // TODO : add api to fetch all the notification from backend to see if it is unaffected - t.Skipped() - return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled()) - }) - assert.True(t, notificationPresent) - }) - - t.Run("remove non-notification label from OBC", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{}) - assert.Nil(t, err) - delete(obc.Labels, "test-label") - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - // check whether existing bucket notification unaffected - notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { - // TODO : add api to fetch all the notification from backend to see if it is unaffected - t.Skipped() - return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled()) - }) - assert.True(t, notificationPresent) - }) - - t.Run("remove notification label from OBC", func(t *testing.T) { - // TODO: add remove notification label support in OBC - t.Skipped() - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{}) - assert.Nil(t, err) - delete(obc.Labels, bucketNotificationLabelPrefix+notificationName) - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - // check whether existing bucket notification uneffected - var notificationPresent bool - for i := 0; i < 4; i++ { - notificationPresent = helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled()) - if !notificationPresent { - break - } - time.Sleep(5 * time.Second) - } - assert.False(t, notificationPresent) - }) - - t.Run("perform s3 operations and confirm notifications are no longer received", func(t *testing.T) { - var s3client *rgw.S3Agent - s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName) - s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName) - s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName) - insecure := objectStore.Spec.IsTLSEnabled() - s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil) - - assert.Nil(t, err) - logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) - - t.Run("put object", func(t *testing.T) { - _, err := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey2, contentType) - assert.Nil(t, err) - }) - - t.Run("check for put bucket notification", func(t *testing.T) { - notificationReceived, err := helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, putEvent, ObjectKey2) - assert.False(t, notificationReceived) - assert.Nil(t, err) - }) - - t.Run("delete objects", func(t *testing.T) { - _, err := s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey1) - assert.Nil(t, err) - }) - - t.Run("check for delete bucket notification", func(t *testing.T) { - notificationReceived, err := helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, deleteEvent, ObjectKey2) - assert.False(t, notificationReceived) - assert.Nil(t, err) - }) - }) - - t.Run("add topic, notification to existing OBC", func(t *testing.T) { - newNotificationName := "new-notification" - newTopicName := "new-topic" - t.Run("create CephBucketTopic: new-topic", func(t *testing.T) { - err := helper.TopicClient.CreateTopic(newTopicName, storeName, httpEndpointService) - assert.Nil(t, err) - created := utils.Retry(12, 2*time.Second, "topic is created", func() bool { - return helper.TopicClient.CheckTopic(newTopicName) - }) - assert.True(t, created) - }) - t.Run("create CephBucketNotification: new-notification", func(t *testing.T) { - err = helper.NotificationClient.CreateNotification(newNotificationName, newTopicName) - assert.Nil(t, err) - created := utils.Retry(12, 2*time.Second, "notification is created", func() bool { - return helper.NotificationClient.CheckNotificationCR(newNotificationName) - }) - assert.True(t, created) - }) - t.Run("add notification label to OBC", func(t *testing.T) { - obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{}) - assert.Nil(t, err) - obc.Labels[bucketNotificationLabelPrefix+newNotificationName] = newNotificationName - _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{}) - assert.Nil(t, err) - }) - t.Run("new-notification should be configured for bucket", func(t *testing.T) { - // check whether bucket notification added - notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { - return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, newNotificationName, helper, objectStore.Spec.IsTLSEnabled()) - }) - assert.True(t, notificationPresent) - }) - t.Run("delete CephBucketNotification: new-notification", func(t *testing.T) { - err = helper.NotificationClient.DeleteNotification(newNotificationName, topicName) - assert.Nil(t, err) - t.Run("check notification removed from backend Bucket", func(t *testing.T) { - // TODO: need to add that support - t.Skipped() - }) - }) - t.Run("delete CephBucketTopic: new-topic", func(t *testing.T) { - err = helper.TopicClient.DeleteTopic(newTopicName, storeName, httpEndpointService) - assert.Nil(t, err) - }) - }) - - t.Run("reverse order of creating notification,topic and adding it to ObjectBucketClaim", func(t *testing.T) { - reverseNotificationName := "reverse-notification" - reverseTopicName := "reverse-topic" - reverseOBCName := "reverse-obc" - reverseBucketName := "reverse-bucket" - i := 0 - - t.Run("create ObjectBucketClaim: reverse-obc", func(t *testing.T) { - err := helper.BucketClient.CreateObcNotification(reverseOBCName, bucketStorageClassName, reverseBucketName, reverseNotificationName, true) - assert.Nil(t, err) - - created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool { - return helper.BucketClient.CheckOBC(reverseOBCName, "bound") - }) - assert.True(t, created) - logger.Info("OBC created successfully") - - var bkt rgw.ObjectBucket - for i = 0; i < 4; i++ { - b, code, err := rgw.GetBucket(rgwcontext, reverseBucketName) - if b != nil && err == nil { - bkt = *b - break - } - logger.Warningf("cannot get bucket %q, retrying... bucket: %v. code: %d, err: %v", reverseBucketName, b, code, err) - logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i) - time.Sleep(5 * time.Second) - } - assert.NotEqual(t, 4, i) - assert.Equal(t, reverseBucketName, bkt.Name) - }) - t.Run("create CephBucketNotification: reverse-notification", func(t *testing.T) { - err = helper.NotificationClient.CreateNotification(reverseNotificationName, reverseTopicName) - assert.Nil(t, err) - created := utils.Retry(12, 2*time.Second, "notification is created", func() bool { - return helper.NotificationClient.CheckNotificationCR(reverseNotificationName) - }) - assert.True(t, created) - }) - t.Run("the notification should not configured for the backend bucket until topic is created", func(t *testing.T) { - // check whether bucket notification added, should fail since topic is not created - // TODO: make below check valid with help of status field in NotificationCR - t.Skipped() - }) - t.Run("create CephBucketTopic: reverse-topic", func(t *testing.T) { - err = helper.TopicClient.CreateTopic(reverseTopicName, storeName, httpEndpointService) - assert.Nil(t, err) - created := utils.Retry(12, 2*time.Second, "topic is created", func() bool { - return helper.TopicClient.CheckTopic(reverseTopicName) - }) - assert.True(t, created) - }) - t.Run("notification should be configured after creating the topic", func(t *testing.T) { - // check whether bucket notification added, should pass since topic got created - notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { - return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, reverseOBCName, reverseBucketName, reverseNotificationName, helper, objectStore.Spec.IsTLSEnabled()) - }) - assert.True(t, notificationPresent) - }) - t.Run("delete CephBucketNotification: reverse-notification", func(t *testing.T) { - err = helper.NotificationClient.DeleteNotification(reverseNotificationName, reverseTopicName) - assert.Nil(t, err) - t.Run("check notification removed from backend Bucket", func(t *testing.T) { - // TODO: need to add that support - t.Skipped() - }) - }) - t.Run("delete CephBucketTopic: reverse-topic", func(t *testing.T) { - err = helper.TopicClient.DeleteTopic(reverseTopicName, storeName, httpEndpointService) - assert.Nil(t, err) - }) - t.Run("delete ObjectBucketClaim: reverse-obc", func(t *testing.T) { - err = helper.BucketClient.DeleteObc(reverseOBCName, bucketStorageClassName, reverseBucketName, maxObject, true) - assert.Nil(t, err) - logger.Info("Checking to see if the obc, secret, and cm have all been deleted") - for i = 0; i < 4 && !helper.BucketClient.CheckOBC(reverseOBCName, "deleted"); i++ { - logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i) - time.Sleep(5 * time.Second) - } - assert.NotEqual(t, 4, i) - logger.Info("ensure OBC bucket was deleted") - var rgwErr int - for i = 0; i < 4; i++ { - _, rgwErr, _ = rgw.GetBucket(rgwcontext, reverseBucketName) - if rgwErr == rgw.RGWErrorNotFound { - break - } - logger.Infof("(%d) check bucket deleted, sleeping for 5 seconds ...", i) - time.Sleep(5 * time.Second) - } - assert.NotEqual(t, 4, i) - assert.Equal(t, rgwErr, rgw.RGWErrorNotFound) - }) - }) - - t.Run("delete ObjectBucketClaim", func(t *testing.T) { - i := 0 - dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true) - assert.Nil(t, dobcErr) - logger.Info("Checking to see if the obc, secret, and cm have all been deleted") - for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ { - logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i) - time.Sleep(5 * time.Second) - } - assert.NotEqual(t, 4, i) - - logger.Info("ensure OBC bucket was deleted") - var rgwErr int - for i = 0; i < 4; i++ { - _, rgwErr, _ = rgw.GetBucket(rgwcontext, bucketname) - if rgwErr == rgw.RGWErrorNotFound { - break - } - logger.Infof("(%d) check bucket deleted, sleeping for 5 seconds ...", i) - time.Sleep(5 * time.Second) - } - assert.NotEqual(t, 4, i) - assert.Equal(t, rgwErr, rgw.RGWErrorNotFound) - - dobErr := helper.BucketClient.DeleteBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete") - assert.Nil(t, dobErr) - }) - - t.Run("delete CephBucketNotification", func(t *testing.T) { - err := helper.NotificationClient.DeleteNotification(notificationName, topicName) - assert.Nil(t, err) - t.Run("check notification removed from backend Bucket", func(t *testing.T) { - // TODO: need to add that support - t.Skipped() - }) - }) - - t.Run("delete CephBucketTopic", func(t *testing.T) { - err := helper.TopicClient.DeleteTopic(topicName, storeName, httpEndpointService) - assert.Nil(t, err) - }) - - t.Run("delete CephObjectStore", func(t *testing.T) { - deleteObjectStore(t, k8sh, namespace, storeName) - assertObjectStoreDeletion(t, k8sh, namespace, storeName) - }) -} diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go index e73bee923..eb008c9cb 100644 --- a/tests/integration/ceph_object_test.go +++ b/tests/integration/ceph_object_test.go @@ -46,6 +46,7 @@ import ( "github.com/rook/rook/tests/framework/utils" bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner" "github.com/rook/rook/tests/integration/object/cosi" + "github.com/rook/rook/tests/integration/object/notification" topickafka "github.com/rook/rook/tests/integration/object/topic/kafka" usercaps "github.com/rook/rook/tests/integration/object/user/caps" userkeys "github.com/rook/rook/tests/integration/object/user/keys" @@ -217,6 +218,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install useropmask.Namespace, usercaps.Namespace, cosi.Namespace, + notification.Namespace, ) defer sharedObjectStore.Destroy() @@ -228,10 +230,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install // the ceph-cosi driver cannot reach a TLS object store endpoint, so this // suite skips itself in the TLS pass cosi.TestCephCOSIDriver(s.T(), k8sh, sharedObjectStore) - - bucketNotificationTestStoreName := "bucket-notification-" + storeName - createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable, swiftAndKeystone) - testBucketNotifications(s, helper, k8sh, namespace, bucketNotificationTestStoreName) + notification.TestBucketNotification(s.T(), k8sh, sharedObjectStore) } func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, settings *installer.TestCephSettings, storeName string, swiftAndKeystone bool) { diff --git a/tests/integration/object/README.md b/tests/integration/object/README.md index fe1c765da..bf329ebd7 100644 --- a/tests/integration/object/README.md +++ b/tests/integration/object/README.md @@ -2,7 +2,7 @@ This tree holds the new-style integration tests for rook's object storage features. It is the model for converting the remaining old-style object tests -(`testObjectStoreOperations`, store lifecycle, bucket notifications); +(`testObjectStoreOperations`, store lifecycle); follow the conventions here exactly so the conversion happens once. ## Execution model @@ -34,11 +34,12 @@ must not collide with std-lib package names (no `io`, no `http`). |---|---|---| | `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling | | `cosi` | `object/cosi` | CephCOSIDriver + COSI bucket provisioning | +| `notification` | `object/notification` | CephBucketNotification HTTP endpoint delivery | | `topic/kafka` | `object/topic` | CephBucketTopic kafka endpoints | | `user/caps` | `object/user` | user capabilities | | `user/keys` | `object/user` | explicit S3 key management | | `user/opmask` | `object/user` | user op_mask | -| reserved: `bucket/rw`, `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents`, `notification/*` | | future conversions | +| reserved: `bucket/rw`, `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions | Shared utilities live under `util/`: @@ -211,5 +212,4 @@ verification; wire the dispatcher; delete the old code; retire | `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | per-OBC S3 client, move `lifecycleCmpOpts` out of the dispatcher | | `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `ready` | | `createCephObjectStore`/`runObjectE2ETestLite`/deletion asserts + zone.json canary | `tests/integration/object/lifecycle` | store create/health/delete helpers, `Sharedstore.Installer()` accessor | -| `testBucketNotifications` | `notification/*` | HTTP endpoint fixture, per-OBC S3 client (the pod-log waiter is now `wait4.Assert/RequirePodLog`) | | upgrade-suite object usage | stays in upgrade suite | switch to typed clients + `wait4`/`ready` | diff --git a/tests/integration/object/notification/notification.go b/tests/integration/object/notification/notification.go new file mode 100644 index 000000000..e6a8785e3 --- /dev/null +++ b/tests/integration/object/notification/notification.go @@ -0,0 +1,502 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package notification + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rgw "github.com/rook/rook/pkg/operator/ceph/object" + "github.com/rook/rook/tests/framework/utils" + "github.com/rook/rook/tests/integration/object/util/client" + "github.com/rook/rook/tests/integration/object/util/fixture" + "github.com/rook/rook/tests/integration/object/util/sharedstore" + "github.com/rook/rook/tests/integration/object/util/wait4" +) + +const sinkPort int32 = 8080 + +const ( + putEvent = "ObjectCreated:Put" + deleteEvent = "ObjectRemoved:Delete" +) + +// httpTopic builds a CephBucketTopic that delivers to an HTTP endpoint. +func httpTopic(name, namespace string, store *cephv1.CephObjectStore, uri string) *cephv1.CephBucketTopic { + return &cephv1.CephBucketTopic{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: cephv1.BucketTopicSpec{ + ObjectStoreName: store.Name, + ObjectStoreNamespace: store.Namespace, + Endpoint: cephv1.TopicEndpointSpec{HTTP: &cephv1.HTTPEndpointSpec{URI: uri}}, + }, + } +} + +// bucketNotification builds a CephBucketNotification for object put and delete events. +func bucketNotification(name, namespace, topic string) *cephv1.CephBucketNotification { + return &cephv1.CephBucketNotification{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: cephv1.BucketNotificationSpec{ + Topic: topic, + Events: []cephv1.BucketNotificationEvent{"s3:ObjectCreated:Put", "s3:ObjectRemoved:Delete"}, + }, + } +} + +// notificationLabel is the OBC label the operator watches to wire a notification to a bucket. +func notificationLabel(notification string) string { + return "bucket-notification-" + notification +} + +// notificationOBC builds an OBC labelled to subscribe its bucket to notification. +func notificationOBC(name, namespace, storageClass, notification string) *bktv1alpha1.ObjectBucketClaim { + return &bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{notificationLabel(notification): notification}, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: name, + StorageClassName: storageClass, + }, + } +} + +const Namespace = "test-notification" + +func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) { + var ( + defaultName = Namespace + objectStore = store.ObjectStore() + + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + } + + sinkName = defaultName + "-sink" + sinkSel = labels.SelectorFromSet(labels.Set{"app": sinkName}) + sinkURI = fmt.Sprintf("http://%s.%s:%d", sinkName, ns.Name, sinkPort) + + storageClass = fixture.StorageClass(defaultName, objectStore) + + topic = httpTopic(defaultName+"-topic", ns.Name, objectStore, sinkURI) + notification = bucketNotification(defaultName+"-notification", ns.Name, topic.Name) + obc = notificationOBC(defaultName+"-obc", ns.Name, storageClass.Name, notification.Name) + + body = "test bucket notification payload" + contentType = "text/plain" + key1 = "obj1" + key2 = "obj2" + + btClient = k8sh.RookClientset.CephV1().CephBucketTopics(ns.Name) + bnClient = k8sh.RookClientset.CephV1().CephBucketNotifications(ns.Name) + obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name) + ) + + t.Run("CephBucketNotification http", func(t *testing.T) { + ctx := t.Context() + + fixture.RequireNamespace(t, k8sh, ns) + + requireHTTPSink(ctx, t, k8sh, ns.Name, sinkName) + + fixture.RequireStorageClass(t, k8sh, storageClass) + + t.Run(fmt.Sprintf("create CephBucketTopic %q", topic.Name), func(t *testing.T) { + // topic creation may be slow right after rgw start up + wait4.RequireCreate(ctx, t, btClient, topic, wait4.BucketTopic, wait4.TimeoutMedium) + }) + + t.Run(fmt.Sprintf("create CephBucketNotification %q", notification.Name), func(t *testing.T) { + _, err := bnClient.Create(ctx, notification, metav1.CreateOptions{}) + require.NoError(t, err) + }) + + s3agent := requireBoundOBC(ctx, t, k8sh, store, obc) + + t.Run(fmt.Sprintf("notification %q is configured on bucket %q", notification.Name, obc.Spec.BucketName), func(t *testing.T) { + requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true) + }) + + t.Run(fmt.Sprintf("put object %q delivers a notification", key1), func(t *testing.T) { + _, err := s3agent.PutObjectInBucket(ctx, obc.Spec.BucketName, body, key1, contentType) + require.NoError(t, err) + + requireNotificationReceived(ctx, t, k8sh, ns.Name, sinkSel, putEvent, key1) + assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, deleteEvent, key1) + }) + + t.Run(fmt.Sprintf("delete object %q delivers a notification", key1), func(t *testing.T) { + _, err := s3agent.DeleteObjectInBucket(ctx, obc.Spec.BucketName, key1) + require.NoError(t, err) + + requireNotificationReceived(ctx, t, k8sh, ns.Name, sinkSel, deleteEvent, key1) + assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, putEvent, key1) + }) + + t.Run("an unrelated OBC label change leaves the notification configured", func(t *testing.T) { + t.Run(fmt.Sprintf("add a non-notification label to obc %q", obc.Name), func(t *testing.T) { + updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) { + l["test-label"] = "test-value" + }) + }) + + requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true) + + t.Run(fmt.Sprintf("remove the non-notification label from obc %q", obc.Name), func(t *testing.T) { + updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) { + delete(l, "test-label") + }) + }) + + requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true) + }) + + t.Run("removing the notification label removes the notification from the bucket", func(t *testing.T) { + t.Run(fmt.Sprintf("remove the notification label from obc %q", obc.Name), func(t *testing.T) { + updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) { + delete(l, notificationLabel(notification.Name)) + }) + }) + + requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, false) + }) + + t.Run("an unsubscribed bucket delivers no notifications", func(t *testing.T) { + t.Run(fmt.Sprintf("put object %q", key2), func(t *testing.T) { + _, err := s3agent.PutObjectInBucket(ctx, obc.Spec.BucketName, body, key2, contentType) + require.NoError(t, err) + + assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, putEvent, key2) + }) + + t.Run(fmt.Sprintf("delete object %q", key2), func(t *testing.T) { + _, err := s3agent.DeleteObjectInBucket(ctx, obc.Spec.BucketName, key2) + require.NoError(t, err) + + assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, deleteEvent, key2) + }) + }) + + t.Run("a notification added to an existing OBC is configured on its bucket", func(t *testing.T) { + addTopic := httpTopic(defaultName+"-topic-add", ns.Name, objectStore, sinkURI) + addNotification := bucketNotification(defaultName+"-notification-add", ns.Name, addTopic.Name) + + t.Run(fmt.Sprintf("create CephBucketTopic %q", addTopic.Name), func(t *testing.T) { + wait4.RequireCreate(ctx, t, btClient, addTopic, wait4.BucketTopic, wait4.TimeoutMedium) + }) + + t.Run(fmt.Sprintf("create CephBucketNotification %q", addNotification.Name), func(t *testing.T) { + _, err := bnClient.Create(ctx, addNotification, metav1.CreateOptions{}) + require.NoError(t, err) + }) + + t.Run(fmt.Sprintf("add the notification label for %q to obc %q", addNotification.Name, obc.Name), func(t *testing.T) { + updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) { + l[notificationLabel(addNotification.Name)] = addNotification.Name + }) + }) + + requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, addNotification.Name, true) + + t.Run(fmt.Sprintf("delete CephBucketNotification %q", addNotification.Name), func(t *testing.T) { + wait4.AssertDelete(ctx, t, bnClient, addNotification.Name, wait4.TimeoutShort) + }) + + t.Run(fmt.Sprintf("delete CephBucketTopic %q", addTopic.Name), func(t *testing.T) { + wait4.AssertDelete(ctx, t, btClient, addTopic.Name, wait4.TimeoutShort) + }) + }) + + t.Run("a notification created before its topic is configured once the topic exists", func(t *testing.T) { + revTopic := httpTopic(defaultName+"-topic-rev", ns.Name, objectStore, sinkURI) + revNotification := bucketNotification(defaultName+"-notification-rev", ns.Name, revTopic.Name) + revOBC := notificationOBC(defaultName+"-obc-rev", ns.Name, storageClass.Name, revNotification.Name) + + revS3 := requireBoundOBC(ctx, t, k8sh, store, revOBC) + + t.Run(fmt.Sprintf("create CephBucketNotification %q before its topic", revNotification.Name), func(t *testing.T) { + _, err := bnClient.Create(ctx, revNotification, metav1.CreateOptions{}) + require.NoError(t, err) + }) + + t.Run(fmt.Sprintf("create CephBucketTopic %q", revTopic.Name), func(t *testing.T) { + wait4.RequireCreate(ctx, t, btClient, revTopic, wait4.BucketTopic, wait4.TimeoutMedium) + }) + + t.Run(fmt.Sprintf("notification %q is configured on bucket %q once its topic exists", revNotification.Name, revOBC.Spec.BucketName), func(t *testing.T) { + requireBucketNotification(ctx, t, revS3, revOBC.Spec.BucketName, revNotification.Name, true) + }) + + t.Run(fmt.Sprintf("delete CephBucketNotification %q", revNotification.Name), func(t *testing.T) { + wait4.AssertDelete(ctx, t, bnClient, revNotification.Name, wait4.TimeoutShort) + }) + + t.Run(fmt.Sprintf("delete CephBucketTopic %q", revTopic.Name), func(t *testing.T) { + wait4.AssertDelete(ctx, t, btClient, revTopic.Name, wait4.TimeoutShort) + }) + + t.Run(fmt.Sprintf("delete obc %q", revOBC.Name), func(t *testing.T) { + deleteOBCAndBucket(ctx, t, k8sh, ns.Name, revOBC.Name) + }) + }) + + t.Run(fmt.Sprintf("delete obc %q", obc.Name), func(t *testing.T) { + deleteOBCAndBucket(ctx, t, k8sh, ns.Name, obc.Name) + }) + + t.Run(fmt.Sprintf("delete CephBucketNotification %q", notification.Name), func(t *testing.T) { + wait4.AssertDelete(ctx, t, bnClient, notification.Name, wait4.TimeoutShort) + }) + + t.Run(fmt.Sprintf("delete CephBucketTopic %q", topic.Name), func(t *testing.T) { + wait4.AssertDelete(ctx, t, btClient, topic.Name, wait4.TimeoutShort) + }) + + t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) { + list, err := obcClient.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 0) + }) + + t.Run(fmt.Sprintf("no CephBucketNotification(s) in ns %q", ns.Name), func(t *testing.T) { + list, err := bnClient.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 0) + }) + + t.Run(fmt.Sprintf("no CephBucketTopic(s) in ns %q", ns.Name), func(t *testing.T) { + list, err := btClient.List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 0) + }) + }) +} + +// requireBoundOBC creates obc, waits for it and its backing ObjectBucket to bind, +// and returns an S3 client for the provisioned bucket. +func requireBoundOBC(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore, obc *bktv1alpha1.ObjectBucketClaim) *rgw.S3Agent { + t.Helper() + + obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc.Namespace) + obClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets() + + var agent *rgw.S3Agent + t.Run(fmt.Sprintf("create obc %q", obc.Name), func(t *testing.T) { + liveOBC := wait4.RequireCreate(ctx, t, obcClient, obc, wait4.OBCBound, wait4.TimeoutLong) + wait4.RequireCondition(ctx, t, obClient, liveOBC.Spec.ObjectBucketName, wait4.OBBound, wait4.TimeoutShort) + + agent = obcS3Agent(ctx, t, k8sh, store, obc.Namespace, obc.Name) + }) + return agent +} + +// deleteOBCAndBucket deletes the OBC and waits for its backing ObjectBucket (and +// thus the rgw bucket) to be garbage-collected by the provisioner. +func deleteOBCAndBucket(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string) { + t.Helper() + + obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace) + obClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets() + + liveOBC, err := obcClient.Get(ctx, name, metav1.GetOptions{}) + require.NoError(t, err) + obName := liveOBC.Spec.ObjectBucketName + + wait4.AssertDelete(ctx, t, obcClient, name, wait4.TimeoutShort) + wait4.AssertAbsent(ctx, t, obClient, obName, wait4.TimeoutShort) +} + +// updateOBCLabels applies mutate to a live OBC's labels and updates it. +func updateOBCLabels(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string, mutate func(map[string]string)) { + t.Helper() + + obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace) + + liveOBC, err := obcClient.Get(ctx, name, metav1.GetOptions{}) + require.NoError(t, err) + + if liveOBC.Labels == nil { + liveOBC.Labels = map[string]string{} + } + mutate(liveOBC.Labels) + + _, err = obcClient.Update(ctx, liveOBC, metav1.UpdateOptions{}) + require.NoError(t, err) +} + +// requireHTTPSink deploys a sample HTTP server that records the bucket +// notifications rgw pushes to it, and waits for it to be ready. It needs no +// explicit teardown: deleting the test namespace removes it. +func requireHTTPSink(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string) { + t.Helper() + + podLabels := map[string]string{"app": name} + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: podLabels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: podLabels}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: name, + // TODO: third party image, see https://github.com/rook/rook/issues/9741 + Image: "quay.io/jthottan/pythonwebserver:latest", + Ports: []corev1.ContainerPort{{ContainerPort: sinkPort}}, + }}, + }, + }, + }, + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: corev1.ServiceSpec{ + Selector: podLabels, + Ports: []corev1.ServicePort{{ + Port: sinkPort, + TargetPort: intstr.FromInt32(sinkPort), + }}, + }, + } + + t.Run(fmt.Sprintf("create notification sink %q", name), func(t *testing.T) { + wait4.RequireCreate(ctx, t, k8sh.Clientset.AppsV1().Deployments(namespace), deployment, + func(d *appsv1.Deployment) bool { return d.Status.ReadyReplicas >= 1 }, wait4.TimeoutLong) + wait4.RequireCreate(ctx, t, k8sh.Clientset.CoreV1().Services(namespace), service, nil, wait4.TimeoutShort) + }) +} + +// obcS3Agent builds an S3 client for the bucket provisioned by obcName, using the +// credentials in the OBC's secret and the shared store's endpoint. +func obcS3Agent(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore, namespace, obcName string) *rgw.S3Agent { + t.Helper() + + secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, obcName, metav1.GetOptions{}) + require.NoError(t, err) + + endpoint, err := client.GetS3Endpoint(store.ObjectStore(), k8sh, store.TLSEnabled()) + require.NoError(t, err) + + agent, err := rgw.NewS3Agent( + string(secret.Data["AWS_ACCESS_KEY_ID"]), + string(secret.Data["AWS_SECRET_ACCESS_KEY"]), + endpoint, true, nil, store.TLSEnabled(), nil) + require.NoError(t, err) + + return agent +} + +// requireBucketNotification waits until notificationName is (want=true) or is not +// (want=false) configured on the rgw bucket, read via the S3 API. +func requireBucketNotification(ctx context.Context, t *testing.T, agent *rgw.S3Agent, bucket, notificationName string, want bool) { + t.Helper() + + desc := fmt.Sprintf("notification %q configured=%v on bucket %q", notificationName, want, bucket) + wait4.RequireEventually(ctx, t, wait4.TimeoutMedium, desc, func(ctx context.Context) error { + conf, err := agent.Client.GetBucketNotificationConfiguration(ctx, &s3.GetBucketNotificationConfigurationInput{ + Bucket: &bucket, + }) + if err != nil { + return err + } + + found := false + for _, tc := range conf.TopicConfigurations { + if tc.Id != nil && *tc.Id == notificationName { + found = true + break + } + } + if found != want { + return fmt.Errorf("notification %q configured=%v, want %v", notificationName, found, want) + } + return nil + }) +} + +// requireNotificationReceived blocks until the sink logs a line for the given +// event on the given object key. +func requireNotificationReceived(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, sel labels.Selector, event, key string) { + t.Helper() + + wait4.RequirePodLog(ctx, t, k8sh, namespace, sel, wait4.TimeoutShort, + func(line string) bool { return strings.Contains(line, event) && strings.Contains(line, key) }, + "notification sink received %s for %q", event, key) +} + +// notificationSettleDelay gives an unwanted notification time to arrive (if it +// were going to) before we assert its absence. +const notificationSettleDelay = 5 * time.Second + +// assertNotificationNotReceived asserts the sink did not recently log the given +// event for the given key. It matches only the most recent log lines: an earlier, +// legitimately-received event for the same key may still be in the full history. +func assertNotificationNotReceived(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, sel labels.Selector, event, key string) { + t.Helper() + + time.Sleep(notificationSettleDelay) + + recent := tailSinkLog(ctx, t, k8sh, namespace, sel) + assert.False(t, strings.Contains(recent, event) && strings.Contains(recent, key), + "notification sink unexpectedly received %s for %q", event, key) +} + +// sinkLogTailLines bounds assertNotificationNotReceived to recent log lines. +const sinkLogTailLines = 5 + +// tailSinkLog returns the last sinkLogTailLines log lines of the first pod +// matching sel. +func tailSinkLog(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, sel labels.Selector) string { + t.Helper() + + pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: sel.String()}) + require.NoError(t, err) + require.NotEmpty(t, pods.Items) + + tail := int64(sinkLogTailLines) + stream, err := k8sh.Clientset.CoreV1().Pods(namespace). + GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{TailLines: &tail}). + Stream(ctx) + require.NoError(t, err) + defer stream.Close() + + out, err := io.ReadAll(stream) + require.NoError(t, err) + return string(out) +}