diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go index 99f94bca7..cfcc61ad4 100644 --- a/tests/framework/clients/bucket.go +++ b/tests/framework/clients/bucket.go @@ -20,7 +20,9 @@ import ( b64 "encoding/base64" "fmt" + "github.com/aws/aws-sdk-go/service/s3" bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" + rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" ) @@ -146,3 +148,33 @@ func (b *BucketOperation) CheckOBMaxObject(obcName, maxobject string) bool { fetchMaxObject, _ := b.k8sh.GetResource("ob", obName, "--output", "jsonpath={.spec.endpoint.additionalConfig.maxObjects}") return maxobject == fetchMaxObject } + +// Checks the bucket notifications set on RGW backend bucket +func (b *BucketOperation) CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint string) bool { + var s3client *rgw.S3Agent + s3AccessKey, _ := b.GetAccessKey(obcName) + s3SecretKey, _ := b.GetSecretKey(obcName) + + //TODO : add TLS check + s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, "", true, nil) + if err != nil { + logger.Errorf("S3 client creation failed with error %v", err) + return false + } + + notifications, err := s3client.Client.GetBucketNotificationConfiguration(&s3.GetBucketNotificationConfigurationRequest{ + Bucket: &bucketname, + }) + if err != nil { + logger.Infof("failed to fetch bucket notifications configuration due to %v", err) + return false + } + logger.Infof("%d bucket notifications found in: %+v", len(notifications.TopicConfigurations), notifications) + for _, notification := range notifications.TopicConfigurations { + if *notification.Id == notificationName { + return true + } + logger.Infof("bucket notifications name mismatch %q != %q", *notification.Id, notificationName) + } + return false +} diff --git a/tests/framework/clients/notification.go b/tests/framework/clients/notification.go index 9beeb1589..ec1e769ba 100644 --- a/tests/framework/clients/notification.go +++ b/tests/framework/clients/notification.go @@ -1,4 +1,3 @@ - /* Copyright 2021 The Rook Authors. All rights reserved. @@ -46,7 +45,8 @@ func (n *NotificationOperation) UpdateNotification(notificationName string, topi } // CheckNotification if notification was set -func (t *NotificationOperation) CheckNotification(notificationName string) bool { +func (t *NotificationOperation) CheckNotificationCR(notificationName string) bool { + // TODO: return result based on reconcile status of the CR const resourceName = "cephbucketnotification" _, err := t.k8sh.GetResource(resourceName, notificationName) if err != nil { diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index b33737887..8d4d71f57 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -32,6 +32,7 @@ import ( "time" "github.com/coreos/pkg/capnslog" + bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned" "github.com/pkg/errors" rookclient "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/clusterd" @@ -56,6 +57,7 @@ type K8sHelper struct { remoteExecutor *exec.RemotePodCommandExecutor Clientset *kubernetes.Clientset RookClientset *rookclient.Clientset + BucketClientset *bktclient.Clientset RunningInCluster bool T func() *testing.T } @@ -94,13 +96,17 @@ func CreateK8sHelper(t func() *testing.T) (*K8sHelper, error) { if err != nil { return nil, fmt.Errorf("failed to get rook clientset. %+v", err) } + bucketClientset, err := bktclient.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to get lib-bucket-provisioner clientset. %+v", err) + } remoteExecutor := &exec.RemotePodCommandExecutor{ ClientSet: clientset, RestClient: config, } - h := &K8sHelper{executor: executor, Clientset: clientset, RookClientset: rookClientset, T: t, remoteExecutor: remoteExecutor} + h := &K8sHelper{executor: executor, Clientset: clientset, RookClientset: rookClientset, BucketClientset: bucketClientset, T: t, remoteExecutor: remoteExecutor} if strings.Contains(config.Host, "//10.") { h.RunningInCluster = true } diff --git a/tests/integration/ceph_bucket_notification_test.go b/tests/integration/ceph_bucket_notification_test.go index 2d0fdf6f6..91b285b1c 100644 --- a/tests/integration/ceph_bucket_notification_test.go +++ b/tests/integration/ceph_bucket_notification_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - "github.com/aws/aws-sdk-go/service/s3" "github.com/rook/rook/pkg/daemon/ceph/client" rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/utils" @@ -29,15 +28,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (s *ObjectSuite) TestBucketNotificationsInOrder() { +func (s *ObjectSuite) TestBucketNotifications() { if utils.IsPlatformOpenShift() { s.T().Skip("bucket notification tests skipped on openshift") } objectStoreServicePrefix = objectStoreServicePrefixUniq - storeName := "test-store" + bucketNotificationLabelPrefix := "bucket-notification-" + storeName := "test-store-bucket-notification" tlsEnable := false namespace := s.settings.Namespace + obcNamespace := "default" helper := s.helper k8sh := s.k8sh logger.Infof("Running on Rook Cluster %s", namespace) @@ -47,20 +48,6 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() { clusterInfo := client.AdminTestClusterInfo(namespace) t := s.T() - t.Run("create CephObjectStoreUser", func(t *testing.T) { - createCephObjectUser(s.Suite, helper, k8sh, namespace, storeName, userid, true, true) - i := 0 - for i = 0; i < 4; i++ { - if helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) { - break - } - logger.Info("waiting 5 more seconds for user secret to exist") - time.Sleep(5 * time.Second) - } - assert.NotEqual(t, 4, i) - }) - logger.Info("object store user created") - context := k8sh.MakeContext() objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{}) assert.Nil(t, err) @@ -70,6 +57,7 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() { notificationName := "my-notification" topicName := "my-topic" httpEndpointService := "my-notification-sink" + s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName) t.Run("create CephBucketTopic", func(t *testing.T) { err := helper.TopicClient.CreateTopic(topicName, storeName, httpEndpointService) @@ -85,7 +73,7 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() { err := helper.NotificationClient.CreateNotification(notificationName, topicName) assert.Nil(t, err) created := utils.Retry(12, 2*time.Second, "notification is created", func() bool { - return helper.NotificationClient.CheckNotification(notificationName) + return helper.NotificationClient.CheckNotificationCR(notificationName) }) assert.True(t, created) logger.Info("CephBucketNotification created successfully") @@ -122,40 +110,205 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() { }) t.Run("check CephBucketNotification created for bucket", func(t *testing.T) { - var s3client *rgw.S3Agent - s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName) - s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName) - s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName) - if objectStore.Spec.IsTLSEnabled() { - s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, rgwcontext.ZoneGroup, true) - } else { - s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, rgwcontext.ZoneGroup, true, nil) - } - - assert.Nil(t, err) - logger.Infof("endpoint (%s) Accesskey (%s) secret (%s) region (%s)", s3endpoint, s3AccessKey, s3SecretKey, rgwcontext.ZoneGroup) - - created := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { - notifications, err := s3client.Client.GetBucketNotificationConfiguration(&s3.GetBucketNotificationConfigurationRequest{ - Bucket: &bucketname, - }) - if err != nil { - logger.Infof("failed to fetch bucket notifications configuration due to %v", err) - return false - } - logger.Infof("%d bucket notifications found in: %v", len(notifications.TopicConfigurations), notifications) - for _, notification := range notifications.TopicConfigurations { - if *notification.Id == notificationName { - return true - } - logger.Infof("bucket notifications name mismatch %q != %q", *notification.Id, notificationName) - } - return false + notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { + return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint) }) - assert.True(t, created) + assert.True(t, notificationPresent) logger.Info("CephBucketNotification created successfully on bucket") }) + t.Run("add non-notification label to OBC", func(t *testing.T) { + obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{}) + assert.Nil(t, err) + obc.Labels["test-label"] = "test-value" + _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{}) + assert.Nil(t, err) + // check whether existing bucket notification unaffected + notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { + // TODO : add api to fetch all the notification from backend to see if it is unaffected + t.Skipped() + return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint) + }) + assert.True(t, notificationPresent) + }) + + t.Run("remove non-notification label from OBC", func(t *testing.T) { + obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{}) + assert.Nil(t, err) + delete(obc.Labels, "test-label") + _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{}) + assert.Nil(t, err) + // check whether existing bucket notification unaffected + notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { + // TODO : add api to fetch all the notification from backend to see if it is unaffected + t.Skipped() + return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint) + }) + assert.True(t, notificationPresent) + }) + + t.Run("remove notification label from OBC", func(t *testing.T) { + // TODO: add remove notification label support in OBC + t.Skipped() + obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{}) + assert.Nil(t, err) + delete(obc.Labels, bucketNotificationLabelPrefix+notificationName) + _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{}) + assert.Nil(t, err) + // check whether existing bucket notification uneffected + var notificationPresent bool + for i := 0; i < 4; i++ { + notificationPresent = helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint) + if !notificationPresent { + break + } + time.Sleep(5 * time.Second) + } + assert.False(t, notificationPresent) + }) + + t.Run("add topic, notification to existing OBC", func(t *testing.T) { + newNotificationName := "new-notification" + newTopicName := "new-topic" + t.Run("create CephBucketTopic: new-topic", func(t *testing.T) { + err := helper.TopicClient.CreateTopic(newTopicName, storeName, httpEndpointService) + assert.Nil(t, err) + created := utils.Retry(12, 2*time.Second, "topic is created", func() bool { + return helper.TopicClient.CheckTopic(newTopicName) + }) + assert.True(t, created) + }) + t.Run("create CephBucketNotification: new-notification", func(t *testing.T) { + err = helper.NotificationClient.CreateNotification(newNotificationName, newTopicName) + assert.Nil(t, err) + created := utils.Retry(12, 2*time.Second, "notification is created", func() bool { + return helper.NotificationClient.CheckNotificationCR(newNotificationName) + }) + assert.True(t, created) + }) + t.Run("add notification label to OBC", func(t *testing.T) { + obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{}) + assert.Nil(t, err) + obc.Labels[bucketNotificationLabelPrefix+newNotificationName] = newNotificationName + _, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{}) + assert.Nil(t, err) + }) + t.Run("new-notification should be configured for bucket", func(t *testing.T) { + // check whether bucket notification added + notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { + return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, newNotificationName, s3endpoint) + }) + assert.True(t, notificationPresent) + }) + t.Run("delete CephBucketNotification: new-notification", func(t *testing.T) { + err = helper.NotificationClient.DeleteNotification(newNotificationName, topicName) + assert.Nil(t, err) + t.Run("check notification removed from backend Bucket", func(t *testing.T) { + // TODO: need to add that support + t.Skipped() + }) + }) + t.Run("delete CephBucketTopic: new-topic", func(t *testing.T) { + err = helper.TopicClient.DeleteTopic(newTopicName, storeName, httpEndpointService) + assert.Nil(t, err) + }) + }) + + t.Run("reverse order of creating notification,topic and adding it to ObjectBucketClaim", func(t *testing.T) { + reverseNotificationName := "reverse-notification" + reverseTopicName := "reverse-topic" + reverseOBCName := "reverse-obc" + reverseBucketName := "reverse-bucket" + i := 0 + + t.Run("create ObjectBucketClaim: reverse-obc", func(t *testing.T) { + err := helper.BucketClient.CreateObcNotification(reverseOBCName, bucketStorageClassName, reverseBucketName, reverseNotificationName, true) + assert.Nil(t, err) + + created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool { + return helper.BucketClient.CheckOBC(reverseOBCName, "bound") + }) + assert.True(t, created) + logger.Info("OBC created successfully") + + var bkt rgw.ObjectBucket + for i = 0; i < 4; i++ { + b, code, err := rgw.GetBucket(rgwcontext, reverseBucketName) + if b != nil && err == nil { + bkt = *b + break + } + logger.Warningf("cannot get bucket %q, retrying... bucket: %v. code: %d, err: %v", reverseBucketName, b, code, err) + logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i) + time.Sleep(5 * time.Second) + } + assert.NotEqual(t, 4, i) + assert.Equal(t, reverseBucketName, bkt.Name) + }) + t.Run("create CephBucketNotification: reverse-notification", func(t *testing.T) { + err = helper.NotificationClient.CreateNotification(reverseNotificationName, reverseTopicName) + assert.Nil(t, err) + created := utils.Retry(12, 2*time.Second, "notification is created", func() bool { + return helper.NotificationClient.CheckNotificationCR(reverseNotificationName) + }) + assert.True(t, created) + }) + t.Run("the notification should not configured for the backend bucket until topic is created", func(t *testing.T) { + // check whether bucket notification added, should fail since topic is not created + // TODO: make below check valid with help of status field in NotificationCR + t.Skipped() + }) + t.Run("create CephBucketTopic: reverse-topic", func(t *testing.T) { + err = helper.TopicClient.CreateTopic(reverseTopicName, storeName, httpEndpointService) + assert.Nil(t, err) + created := utils.Retry(12, 2*time.Second, "topic is created", func() bool { + return helper.TopicClient.CheckTopic(reverseTopicName) + }) + assert.True(t, created) + }) + t.Run("notification should be configured after creating the topic", func(t *testing.T) { + // check whether bucket notification added, should pass since topic got created + notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool { + return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, reverseOBCName, reverseBucketName, reverseNotificationName, s3endpoint) + }) + assert.True(t, notificationPresent) + }) + t.Run("delete CephBucketNotification: reverse-notification", func(t *testing.T) { + err = helper.NotificationClient.DeleteNotification(reverseNotificationName, reverseTopicName) + assert.Nil(t, err) + t.Run("check notification removed from backend Bucket", func(t *testing.T) { + // TODO: need to add that support + t.Skipped() + }) + }) + t.Run("delete CephBucketTopic: reverse-topic", func(t *testing.T) { + err = helper.TopicClient.DeleteTopic(reverseTopicName, storeName, httpEndpointService) + assert.Nil(t, err) + }) + t.Run("delete ObjectBucketClaim: reverse-obc", func(t *testing.T) { + err = helper.BucketClient.DeleteObc(reverseOBCName, bucketStorageClassName, reverseBucketName, maxObject, true) + assert.Nil(t, err) + logger.Info("Checking to see if the obc, secret, and cm have all been deleted") + for i = 0; i < 4 && !helper.BucketClient.CheckOBC(reverseOBCName, "deleted"); i++ { + logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i) + time.Sleep(5 * time.Second) + } + assert.NotEqual(t, 4, i) + logger.Info("ensure OBC bucket was deleted") + var rgwErr int + for i = 0; i < 4; i++ { + _, rgwErr, _ = rgw.GetBucket(rgwcontext, reverseBucketName) + if rgwErr == rgw.RGWErrorNotFound { + break + } + logger.Infof("(%d) check bucket deleted, sleeping for 5 seconds ...", i) + time.Sleep(5 * time.Second) + } + assert.NotEqual(t, 4, i) + assert.Equal(t, rgwErr, rgw.RGWErrorNotFound) + }) + }) + t.Run("delete ObjectBucketClaim", func(t *testing.T) { i := 0 dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true) @@ -187,6 +340,10 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() { t.Run("delete CephBucketNotification", func(t *testing.T) { err := helper.NotificationClient.DeleteNotification(notificationName, topicName) assert.Nil(t, err) + t.Run("check notification removed from backend Bucket", func(t *testing.T) { + // TODO: need to add that support + t.Skipped() + }) }) t.Run("delete CephBucketTopic", func(t *testing.T) { @@ -194,19 +351,6 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() { assert.Nil(t, err) }) - t.Run("delete CephObjectStoreUser", func(t *testing.T) { - dosuErr := helper.ObjectUserClient.Delete(namespace, userid) - assert.Nil(t, dosuErr) - logger.Info("Object store user deleted successfully") - logger.Info("Checking to see if the user secret has been deleted") - i := 0 - for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == true; i++ { - logger.Infof("(%d) secret check sleeping for 5 seconds ...", i) - time.Sleep(5 * time.Second) - } - assert.False(t, helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid)) - }) - t.Run("delete CephObjectStore", func(t *testing.T) { deleteObjectStore(t, k8sh, namespace, storeName) assertObjectStoreDeletion(t, k8sh, namespace, storeName)