Merge pull request #9169 from thotz/addtestcasesbucketnotification

test: add more test cases for bucket notfication
This commit is contained in:
Blaine Gardner authored and GitHub committed 2021-12-09 08:54:21 -07:00
commit 2467daa406
4 files changed
+246 -64

No files matched your search

+32
View File
@@ -20,7 +20,9 @@ import (
b64 "encoding/base64"
"fmt"
"github.com/aws/aws-sdk-go/service/s3"
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
rgw "github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
)
@@ -146,3 +148,33 @@ func (b *BucketOperation) CheckOBMaxObject(obcName, maxobject string) bool {
fetchMaxObject, _ := b.k8sh.GetResource("ob", obName, "--output", "jsonpath={.spec.endpoint.additionalConfig.maxObjects}")
return maxobject == fetchMaxObject
}
// Checks the bucket notifications set on RGW backend bucket
func (b *BucketOperation) CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint string) bool {
var s3client *rgw.S3Agent
s3AccessKey, _ := b.GetAccessKey(obcName)
s3SecretKey, _ := b.GetSecretKey(obcName)
//TODO : add TLS check
s3client, err := rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, "", true, nil)
if err != nil {
logger.Errorf("S3 client creation failed with error %v", err)
return false
}
notifications, err := s3client.Client.GetBucketNotificationConfiguration(&s3.GetBucketNotificationConfigurationRequest{
Bucket: &bucketname,
})
if err != nil {
logger.Infof("failed to fetch bucket notifications configuration due to %v", err)
return false
}
logger.Infof("%d bucket notifications found in: %+v", len(notifications.TopicConfigurations), notifications)
for _, notification := range notifications.TopicConfigurations {
if *notification.Id == notificationName {
return true
}
logger.Infof("bucket notifications name mismatch %q != %q", *notification.Id, notificationName)
}
return false
}
+2 -2
View File
@@ -1,4 +1,3 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
@@ -46,7 +45,8 @@ func (n *NotificationOperation) UpdateNotification(notificationName string, topi
}
// CheckNotification if notification was set
func (t *NotificationOperation) CheckNotification(notificationName string) bool {
func (t *NotificationOperation) CheckNotificationCR(notificationName string) bool {
// TODO: return result based on reconcile status of the CR
const resourceName = "cephbucketnotification"
_, err := t.k8sh.GetResource(resourceName, notificationName)
if err != nil {
+7 -1
View File
@@ -32,6 +32,7 @@ import (
"time"
"github.com/coreos/pkg/capnslog"
bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned"
"github.com/pkg/errors"
rookclient "github.com/rook/rook/pkg/client/clientset/versioned"
"github.com/rook/rook/pkg/clusterd"
@@ -56,6 +57,7 @@ type K8sHelper struct {
remoteExecutor *exec.RemotePodCommandExecutor
Clientset *kubernetes.Clientset
RookClientset *rookclient.Clientset
BucketClientset *bktclient.Clientset
RunningInCluster bool
T func() *testing.T
}
@@ -94,13 +96,17 @@ func CreateK8sHelper(t func() *testing.T) (*K8sHelper, error) {
if err != nil {
return nil, fmt.Errorf("failed to get rook clientset. %+v", err)
}
bucketClientset, err := bktclient.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("failed to get lib-bucket-provisioner clientset. %+v", err)
}
remoteExecutor := &exec.RemotePodCommandExecutor{
ClientSet: clientset,
RestClient: config,
}
h := &K8sHelper{executor: executor, Clientset: clientset, RookClientset: rookClientset, T: t, remoteExecutor: remoteExecutor}
h := &K8sHelper{executor: executor, Clientset: clientset, RookClientset: rookClientset, BucketClientset: bucketClientset, T: t, remoteExecutor: remoteExecutor}
if strings.Contains(config.Host, "//10.") {
h.RunningInCluster = true
}
@@ -21,7 +21,6 @@ import (
"testing"
"time"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/rook/rook/pkg/daemon/ceph/client"
rgw "github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/tests/framework/utils"
@@ -29,15 +28,17 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (s *ObjectSuite) TestBucketNotificationsInOrder() {
func (s *ObjectSuite) TestBucketNotifications() {
if utils.IsPlatformOpenShift() {
s.T().Skip("bucket notification tests skipped on openshift")
}
objectStoreServicePrefix = objectStoreServicePrefixUniq
storeName := "test-store"
bucketNotificationLabelPrefix := "bucket-notification-"
storeName := "test-store-bucket-notification"
tlsEnable := false
namespace := s.settings.Namespace
obcNamespace := "default"
helper := s.helper
k8sh := s.k8sh
logger.Infof("Running on Rook Cluster %s", namespace)
@@ -47,20 +48,6 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() {
clusterInfo := client.AdminTestClusterInfo(namespace)
t := s.T()
t.Run("create CephObjectStoreUser", func(t *testing.T) {
createCephObjectUser(s.Suite, helper, k8sh, namespace, storeName, userid, true, true)
i := 0
for i = 0; i < 4; i++ {
if helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) {
break
}
logger.Info("waiting 5 more seconds for user secret to exist")
time.Sleep(5 * time.Second)
}
assert.NotEqual(t, 4, i)
})
logger.Info("object store user created")
context := k8sh.MakeContext()
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{})
assert.Nil(t, err)
@@ -70,6 +57,7 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() {
notificationName := "my-notification"
topicName := "my-topic"
httpEndpointService := "my-notification-sink"
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
t.Run("create CephBucketTopic", func(t *testing.T) {
err := helper.TopicClient.CreateTopic(topicName, storeName, httpEndpointService)
@@ -85,7 +73,7 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() {
err := helper.NotificationClient.CreateNotification(notificationName, topicName)
assert.Nil(t, err)
created := utils.Retry(12, 2*time.Second, "notification is created", func() bool {
return helper.NotificationClient.CheckNotification(notificationName)
return helper.NotificationClient.CheckNotificationCR(notificationName)
})
assert.True(t, created)
logger.Info("CephBucketNotification created successfully")
@@ -122,40 +110,205 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() {
})
t.Run("check CephBucketNotification created for bucket", func(t *testing.T) {
var s3client *rgw.S3Agent
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
if objectStore.Spec.IsTLSEnabled() {
s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, rgwcontext.ZoneGroup, true)
} else {
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, rgwcontext.ZoneGroup, true, nil)
}
assert.Nil(t, err)
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s) region (%s)", s3endpoint, s3AccessKey, s3SecretKey, rgwcontext.ZoneGroup)
created := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
notifications, err := s3client.Client.GetBucketNotificationConfiguration(&s3.GetBucketNotificationConfigurationRequest{
Bucket: &bucketname,
})
if err != nil {
logger.Infof("failed to fetch bucket notifications configuration due to %v", err)
return false
}
logger.Infof("%d bucket notifications found in: %v", len(notifications.TopicConfigurations), notifications)
for _, notification := range notifications.TopicConfigurations {
if *notification.Id == notificationName {
return true
}
logger.Infof("bucket notifications name mismatch %q != %q", *notification.Id, notificationName)
}
return false
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint)
})
assert.True(t, created)
assert.True(t, notificationPresent)
logger.Info("CephBucketNotification created successfully on bucket")
})
t.Run("add non-notification label to OBC", func(t *testing.T) {
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{})
assert.Nil(t, err)
obc.Labels["test-label"] = "test-value"
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{})
assert.Nil(t, err)
// check whether existing bucket notification unaffected
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
// TODO : add api to fetch all the notification from backend to see if it is unaffected
t.Skipped()
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint)
})
assert.True(t, notificationPresent)
})
t.Run("remove non-notification label from OBC", func(t *testing.T) {
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{})
assert.Nil(t, err)
delete(obc.Labels, "test-label")
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{})
assert.Nil(t, err)
// check whether existing bucket notification unaffected
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
// TODO : add api to fetch all the notification from backend to see if it is unaffected
t.Skipped()
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint)
})
assert.True(t, notificationPresent)
})
t.Run("remove notification label from OBC", func(t *testing.T) {
// TODO: add remove notification label support in OBC
t.Skipped()
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{})
assert.Nil(t, err)
delete(obc.Labels, bucketNotificationLabelPrefix+notificationName)
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{})
assert.Nil(t, err)
// check whether existing bucket notification uneffected
var notificationPresent bool
for i := 0; i < 4; i++ {
notificationPresent = helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, s3endpoint)
if !notificationPresent {
break
}
time.Sleep(5 * time.Second)
}
assert.False(t, notificationPresent)
})
t.Run("add topic, notification to existing OBC", func(t *testing.T) {
newNotificationName := "new-notification"
newTopicName := "new-topic"
t.Run("create CephBucketTopic: new-topic", func(t *testing.T) {
err := helper.TopicClient.CreateTopic(newTopicName, storeName, httpEndpointService)
assert.Nil(t, err)
created := utils.Retry(12, 2*time.Second, "topic is created", func() bool {
return helper.TopicClient.CheckTopic(newTopicName)
})
assert.True(t, created)
})
t.Run("create CephBucketNotification: new-notification", func(t *testing.T) {
err = helper.NotificationClient.CreateNotification(newNotificationName, newTopicName)
assert.Nil(t, err)
created := utils.Retry(12, 2*time.Second, "notification is created", func() bool {
return helper.NotificationClient.CheckNotificationCR(newNotificationName)
})
assert.True(t, created)
})
t.Run("add notification label to OBC", func(t *testing.T) {
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{})
assert.Nil(t, err)
obc.Labels[bucketNotificationLabelPrefix+newNotificationName] = newNotificationName
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{})
assert.Nil(t, err)
})
t.Run("new-notification should be configured for bucket", func(t *testing.T) {
// check whether bucket notification added
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, newNotificationName, s3endpoint)
})
assert.True(t, notificationPresent)
})
t.Run("delete CephBucketNotification: new-notification", func(t *testing.T) {
err = helper.NotificationClient.DeleteNotification(newNotificationName, topicName)
assert.Nil(t, err)
t.Run("check notification removed from backend Bucket", func(t *testing.T) {
// TODO: need to add that support
t.Skipped()
})
})
t.Run("delete CephBucketTopic: new-topic", func(t *testing.T) {
err = helper.TopicClient.DeleteTopic(newTopicName, storeName, httpEndpointService)
assert.Nil(t, err)
})
})
t.Run("reverse order of creating notification,topic and adding it to ObjectBucketClaim", func(t *testing.T) {
reverseNotificationName := "reverse-notification"
reverseTopicName := "reverse-topic"
reverseOBCName := "reverse-obc"
reverseBucketName := "reverse-bucket"
i := 0
t.Run("create ObjectBucketClaim: reverse-obc", func(t *testing.T) {
err := helper.BucketClient.CreateObcNotification(reverseOBCName, bucketStorageClassName, reverseBucketName, reverseNotificationName, true)
assert.Nil(t, err)
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
return helper.BucketClient.CheckOBC(reverseOBCName, "bound")
})
assert.True(t, created)
logger.Info("OBC created successfully")
var bkt rgw.ObjectBucket
for i = 0; i < 4; i++ {
b, code, err := rgw.GetBucket(rgwcontext, reverseBucketName)
if b != nil && err == nil {
bkt = *b
break
}
logger.Warningf("cannot get bucket %q, retrying... bucket: %v. code: %d, err: %v", reverseBucketName, b, code, err)
logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i)
time.Sleep(5 * time.Second)
}
assert.NotEqual(t, 4, i)
assert.Equal(t, reverseBucketName, bkt.Name)
})
t.Run("create CephBucketNotification: reverse-notification", func(t *testing.T) {
err = helper.NotificationClient.CreateNotification(reverseNotificationName, reverseTopicName)
assert.Nil(t, err)
created := utils.Retry(12, 2*time.Second, "notification is created", func() bool {
return helper.NotificationClient.CheckNotificationCR(reverseNotificationName)
})
assert.True(t, created)
})
t.Run("the notification should not configured for the backend bucket until topic is created", func(t *testing.T) {
// check whether bucket notification added, should fail since topic is not created
// TODO: make below check valid with help of status field in NotificationCR
t.Skipped()
})
t.Run("create CephBucketTopic: reverse-topic", func(t *testing.T) {
err = helper.TopicClient.CreateTopic(reverseTopicName, storeName, httpEndpointService)
assert.Nil(t, err)
created := utils.Retry(12, 2*time.Second, "topic is created", func() bool {
return helper.TopicClient.CheckTopic(reverseTopicName)
})
assert.True(t, created)
})
t.Run("notification should be configured after creating the topic", func(t *testing.T) {
// check whether bucket notification added, should pass since topic got created
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, reverseOBCName, reverseBucketName, reverseNotificationName, s3endpoint)
})
assert.True(t, notificationPresent)
})
t.Run("delete CephBucketNotification: reverse-notification", func(t *testing.T) {
err = helper.NotificationClient.DeleteNotification(reverseNotificationName, reverseTopicName)
assert.Nil(t, err)
t.Run("check notification removed from backend Bucket", func(t *testing.T) {
// TODO: need to add that support
t.Skipped()
})
})
t.Run("delete CephBucketTopic: reverse-topic", func(t *testing.T) {
err = helper.TopicClient.DeleteTopic(reverseTopicName, storeName, httpEndpointService)
assert.Nil(t, err)
})
t.Run("delete ObjectBucketClaim: reverse-obc", func(t *testing.T) {
err = helper.BucketClient.DeleteObc(reverseOBCName, bucketStorageClassName, reverseBucketName, maxObject, true)
assert.Nil(t, err)
logger.Info("Checking to see if the obc, secret, and cm have all been deleted")
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(reverseOBCName, "deleted"); i++ {
logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i)
time.Sleep(5 * time.Second)
}
assert.NotEqual(t, 4, i)
logger.Info("ensure OBC bucket was deleted")
var rgwErr int
for i = 0; i < 4; i++ {
_, rgwErr, _ = rgw.GetBucket(rgwcontext, reverseBucketName)
if rgwErr == rgw.RGWErrorNotFound {
break
}
logger.Infof("(%d) check bucket deleted, sleeping for 5 seconds ...", i)
time.Sleep(5 * time.Second)
}
assert.NotEqual(t, 4, i)
assert.Equal(t, rgwErr, rgw.RGWErrorNotFound)
})
})
t.Run("delete ObjectBucketClaim", func(t *testing.T) {
i := 0
dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
@@ -187,6 +340,10 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() {
t.Run("delete CephBucketNotification", func(t *testing.T) {
err := helper.NotificationClient.DeleteNotification(notificationName, topicName)
assert.Nil(t, err)
t.Run("check notification removed from backend Bucket", func(t *testing.T) {
// TODO: need to add that support
t.Skipped()
})
})
t.Run("delete CephBucketTopic", func(t *testing.T) {
@@ -194,19 +351,6 @@ func (s *ObjectSuite) TestBucketNotificationsInOrder() {
assert.Nil(t, err)
})
t.Run("delete CephObjectStoreUser", func(t *testing.T) {
dosuErr := helper.ObjectUserClient.Delete(namespace, userid)
assert.Nil(t, dosuErr)
logger.Info("Object store user deleted successfully")
logger.Info("Checking to see if the user secret has been deleted")
i := 0
for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == true; i++ {
logger.Infof("(%d) secret check sleeping for 5 seconds ...", i)
time.Sleep(5 * time.Second)
}
assert.False(t, helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))
})
t.Run("delete CephObjectStore", func(t *testing.T) {
deleteObjectStore(t, k8sh, namespace, storeName)
assertObjectStoreDeletion(t, k8sh, namespace, storeName)