test: convert the bucket notification suite to the shared store
Rework the bucket notification integration test onto the shared CephObjectStore fixture and the wait4 toolkit, following the object suite conversion playbook. The old test stood up its own object store and drove everything through tests/framework/clients and kubectl with fixed-interval polling. The new tests/integration/object/notification package is an ordered t.Run script on typed clients and watch-based waits. It deploys an HTTP sink, drives a CephBucketTopic (HTTP endpoint), CephBucketNotification, and a notification-labelled OBC, and verifies both end-to-end delivery (matched in the sink's logs) and the notification configured on the rgw bucket (read over the S3 API), covering the label add/remove and notification-before-topic orderings. Running against the shared store, it no longer creates a dedicated object store and now exercises both the TLS and non-TLS suite passes. Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
This commit is contained in:
1 parent
6bfe0bf45c
commit
780f0cceb0
4 files changed
+508
-450
No files matched your search
@@ -1,443 +0,0 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
rgw "github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
httpServerName = "sample-http-server"
|
||||
httpServerNameSpace = "default"
|
||||
httpServerPort = "8080"
|
||||
httpEndpointService = "http://" + httpServerName + "." + httpServerNameSpace + ":" + httpServerPort
|
||||
putEvent = "ObjectCreated:Put"
|
||||
deleteEvent = "ObjectRemoved:Delete"
|
||||
)
|
||||
|
||||
func testBucketNotifications(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) {
|
||||
if utils.IsPlatformOpenShift() {
|
||||
s.T().Skip("bucket notification tests skipped on openshift")
|
||||
}
|
||||
|
||||
bucketNotificationLabelPrefix := "bucket-notification-"
|
||||
obcNamespace := "default"
|
||||
|
||||
ctx := context.TODO()
|
||||
clusterInfo := client.AdminTestClusterInfo(namespace)
|
||||
t := s.T()
|
||||
|
||||
context := k8sh.MakeContext()
|
||||
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
rgwcontext, err := rgw.NewMultisiteContext(context, clusterInfo, objectStore)
|
||||
assert.Nil(t, err)
|
||||
|
||||
notificationName := "my-notification"
|
||||
topicName := "my-topic"
|
||||
appLabel := "app=" + httpServerName
|
||||
|
||||
logger.Infof("Testing Bucket Notifications on %s", storeName)
|
||||
|
||||
t.Run("create HTTP Endpoint for receiving notifications", func(t *testing.T) {
|
||||
err := helper.TopicClient.CreateHTTPServer(httpServerName, httpServerNameSpace, httpServerPort)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("create CephBucketTopic", func(t *testing.T) {
|
||||
err := helper.TopicClient.CreateTopic(topicName, storeName, httpEndpointService)
|
||||
assert.Nil(t, err)
|
||||
created := utils.Retry(30, 5*time.Second, "topic is created", func() bool {
|
||||
return helper.TopicClient.CheckTopic(topicName)
|
||||
})
|
||||
assert.True(t, created)
|
||||
logger.Info("CephBucketTopic created successfully")
|
||||
})
|
||||
|
||||
t.Run("create CephBucketNotification", func(t *testing.T) {
|
||||
err := helper.NotificationClient.CreateNotification(notificationName, topicName)
|
||||
assert.Nil(t, err)
|
||||
created := utils.Retry(12, 2*time.Second, "notification is created", func() bool {
|
||||
return helper.NotificationClient.CheckNotificationCR(notificationName)
|
||||
})
|
||||
assert.True(t, created)
|
||||
logger.Info("CephBucketNotification created successfully")
|
||||
})
|
||||
|
||||
t.Run("create ObjectBucketClaim", func(t *testing.T) {
|
||||
logger.Infof("create OBC %q with storageclass %q and notification %q", obcName, bucketStorageClassName, notificationName)
|
||||
cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete")
|
||||
assert.Nil(t, cobErr)
|
||||
cobcErr := helper.BucketClient.CreateObcNotification(obcName, bucketStorageClassName, bucketname, notificationName, true)
|
||||
assert.Nil(t, cobcErr)
|
||||
|
||||
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
|
||||
return helper.BucketClient.CheckOBC(obcName, "bound")
|
||||
})
|
||||
assert.True(t, created)
|
||||
logger.Info("OBC created successfully")
|
||||
|
||||
var bkt rgw.ObjectBucket
|
||||
i := 0
|
||||
for i = 0; i < 4; i++ {
|
||||
b, code, err := rgw.GetBucket(rgwcontext, bucketname)
|
||||
if b != nil && err == nil {
|
||||
bkt = *b
|
||||
break
|
||||
}
|
||||
logger.Warningf("cannot get bucket %q, retrying... bucket: %v. code: %d, err: %v", bucketname, b, code, err)
|
||||
logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.NotEqual(t, 4, i)
|
||||
assert.Equal(t, bucketname, bkt.Name)
|
||||
logger.Info("OBC, Secret and ConfigMap created")
|
||||
})
|
||||
|
||||
t.Run("perform s3 operations and check for notifications", func(t *testing.T) {
|
||||
var s3client *rgw.S3Agent
|
||||
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
|
||||
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
|
||||
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
|
||||
insecure := objectStore.Spec.IsTLSEnabled()
|
||||
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
|
||||
|
||||
assert.Nil(t, err)
|
||||
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
|
||||
|
||||
t.Run("put object", func(t *testing.T) {
|
||||
_, err := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey1, contentType)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("check for put bucket notification", func(t *testing.T) {
|
||||
notificationReceived, err := helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, putEvent, ObjectKey1)
|
||||
assert.True(t, notificationReceived)
|
||||
assert.Nil(t, err)
|
||||
// negative test case to confirm didn't receive any delete event notification
|
||||
notificationReceived, err = helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, deleteEvent, ObjectKey1)
|
||||
assert.False(t, notificationReceived)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("delete objects", func(t *testing.T) {
|
||||
_, err := s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey1)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("check for delete bucket notification", func(t *testing.T) {
|
||||
notificationReceived, err := helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, deleteEvent, ObjectKey1)
|
||||
assert.True(t, notificationReceived)
|
||||
assert.Nil(t, err)
|
||||
// negative test case to confirm didn't receive any put event notification
|
||||
notificationReceived, err = helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, putEvent, ObjectKey1)
|
||||
assert.False(t, notificationReceived)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("check CephBucketNotification created for bucket", func(t *testing.T) {
|
||||
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
|
||||
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled())
|
||||
})
|
||||
assert.True(t, notificationPresent)
|
||||
logger.Info("CephBucketNotification created successfully on bucket")
|
||||
})
|
||||
|
||||
t.Run("add non-notification label to OBC", func(t *testing.T) {
|
||||
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
obc.Labels["test-label"] = "test-value"
|
||||
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{})
|
||||
assert.Nil(t, err)
|
||||
// check whether existing bucket notification unaffected
|
||||
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
|
||||
// TODO : add api to fetch all the notification from backend to see if it is unaffected
|
||||
t.Skipped()
|
||||
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled())
|
||||
})
|
||||
assert.True(t, notificationPresent)
|
||||
})
|
||||
|
||||
t.Run("remove non-notification label from OBC", func(t *testing.T) {
|
||||
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
delete(obc.Labels, "test-label")
|
||||
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{})
|
||||
assert.Nil(t, err)
|
||||
// check whether existing bucket notification unaffected
|
||||
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
|
||||
// TODO : add api to fetch all the notification from backend to see if it is unaffected
|
||||
t.Skipped()
|
||||
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled())
|
||||
})
|
||||
assert.True(t, notificationPresent)
|
||||
})
|
||||
|
||||
t.Run("remove notification label from OBC", func(t *testing.T) {
|
||||
// TODO: add remove notification label support in OBC
|
||||
t.Skipped()
|
||||
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
delete(obc.Labels, bucketNotificationLabelPrefix+notificationName)
|
||||
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{})
|
||||
assert.Nil(t, err)
|
||||
// check whether existing bucket notification uneffected
|
||||
var notificationPresent bool
|
||||
for i := 0; i < 4; i++ {
|
||||
notificationPresent = helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, notificationName, helper, objectStore.Spec.IsTLSEnabled())
|
||||
if !notificationPresent {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.False(t, notificationPresent)
|
||||
})
|
||||
|
||||
t.Run("perform s3 operations and confirm notifications are no longer received", func(t *testing.T) {
|
||||
var s3client *rgw.S3Agent
|
||||
s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
|
||||
s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
|
||||
s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
|
||||
insecure := objectStore.Spec.IsTLSEnabled()
|
||||
s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
|
||||
|
||||
assert.Nil(t, err)
|
||||
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
|
||||
|
||||
t.Run("put object", func(t *testing.T) {
|
||||
_, err := s3client.PutObjectInBucket(ctx, bucketname, ObjBody, ObjectKey2, contentType)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("check for put bucket notification", func(t *testing.T) {
|
||||
notificationReceived, err := helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, putEvent, ObjectKey2)
|
||||
assert.False(t, notificationReceived)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("delete objects", func(t *testing.T) {
|
||||
_, err := s3client.DeleteObjectInBucket(ctx, bucketname, ObjectKey1)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("check for delete bucket notification", func(t *testing.T) {
|
||||
notificationReceived, err := helper.NotificationClient.CheckNotificationFromHTTPEndPoint(appLabel, deleteEvent, ObjectKey2)
|
||||
assert.False(t, notificationReceived)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("add topic, notification to existing OBC", func(t *testing.T) {
|
||||
newNotificationName := "new-notification"
|
||||
newTopicName := "new-topic"
|
||||
t.Run("create CephBucketTopic: new-topic", func(t *testing.T) {
|
||||
err := helper.TopicClient.CreateTopic(newTopicName, storeName, httpEndpointService)
|
||||
assert.Nil(t, err)
|
||||
created := utils.Retry(12, 2*time.Second, "topic is created", func() bool {
|
||||
return helper.TopicClient.CheckTopic(newTopicName)
|
||||
})
|
||||
assert.True(t, created)
|
||||
})
|
||||
t.Run("create CephBucketNotification: new-notification", func(t *testing.T) {
|
||||
err = helper.NotificationClient.CreateNotification(newNotificationName, newTopicName)
|
||||
assert.Nil(t, err)
|
||||
created := utils.Retry(12, 2*time.Second, "notification is created", func() bool {
|
||||
return helper.NotificationClient.CheckNotificationCR(newNotificationName)
|
||||
})
|
||||
assert.True(t, created)
|
||||
})
|
||||
t.Run("add notification label to OBC", func(t *testing.T) {
|
||||
obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Get(ctx, obcName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
obc.Labels[bucketNotificationLabelPrefix+newNotificationName] = newNotificationName
|
||||
_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obcNamespace).Update(ctx, obc, metav1.UpdateOptions{})
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
t.Run("new-notification should be configured for bucket", func(t *testing.T) {
|
||||
// check whether bucket notification added
|
||||
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
|
||||
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, obcName, bucketname, newNotificationName, helper, objectStore.Spec.IsTLSEnabled())
|
||||
})
|
||||
assert.True(t, notificationPresent)
|
||||
})
|
||||
t.Run("delete CephBucketNotification: new-notification", func(t *testing.T) {
|
||||
err = helper.NotificationClient.DeleteNotification(newNotificationName, topicName)
|
||||
assert.Nil(t, err)
|
||||
t.Run("check notification removed from backend Bucket", func(t *testing.T) {
|
||||
// TODO: need to add that support
|
||||
t.Skipped()
|
||||
})
|
||||
})
|
||||
t.Run("delete CephBucketTopic: new-topic", func(t *testing.T) {
|
||||
err = helper.TopicClient.DeleteTopic(newTopicName, storeName, httpEndpointService)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("reverse order of creating notification,topic and adding it to ObjectBucketClaim", func(t *testing.T) {
|
||||
reverseNotificationName := "reverse-notification"
|
||||
reverseTopicName := "reverse-topic"
|
||||
reverseOBCName := "reverse-obc"
|
||||
reverseBucketName := "reverse-bucket"
|
||||
i := 0
|
||||
|
||||
t.Run("create ObjectBucketClaim: reverse-obc", func(t *testing.T) {
|
||||
err := helper.BucketClient.CreateObcNotification(reverseOBCName, bucketStorageClassName, reverseBucketName, reverseNotificationName, true)
|
||||
assert.Nil(t, err)
|
||||
|
||||
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
|
||||
return helper.BucketClient.CheckOBC(reverseOBCName, "bound")
|
||||
})
|
||||
assert.True(t, created)
|
||||
logger.Info("OBC created successfully")
|
||||
|
||||
var bkt rgw.ObjectBucket
|
||||
for i = 0; i < 4; i++ {
|
||||
b, code, err := rgw.GetBucket(rgwcontext, reverseBucketName)
|
||||
if b != nil && err == nil {
|
||||
bkt = *b
|
||||
break
|
||||
}
|
||||
logger.Warningf("cannot get bucket %q, retrying... bucket: %v. code: %d, err: %v", reverseBucketName, b, code, err)
|
||||
logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.NotEqual(t, 4, i)
|
||||
assert.Equal(t, reverseBucketName, bkt.Name)
|
||||
})
|
||||
t.Run("create CephBucketNotification: reverse-notification", func(t *testing.T) {
|
||||
err = helper.NotificationClient.CreateNotification(reverseNotificationName, reverseTopicName)
|
||||
assert.Nil(t, err)
|
||||
created := utils.Retry(12, 2*time.Second, "notification is created", func() bool {
|
||||
return helper.NotificationClient.CheckNotificationCR(reverseNotificationName)
|
||||
})
|
||||
assert.True(t, created)
|
||||
})
|
||||
t.Run("the notification should not configured for the backend bucket until topic is created", func(t *testing.T) {
|
||||
// check whether bucket notification added, should fail since topic is not created
|
||||
// TODO: make below check valid with help of status field in NotificationCR
|
||||
t.Skipped()
|
||||
})
|
||||
t.Run("create CephBucketTopic: reverse-topic", func(t *testing.T) {
|
||||
err = helper.TopicClient.CreateTopic(reverseTopicName, storeName, httpEndpointService)
|
||||
assert.Nil(t, err)
|
||||
created := utils.Retry(12, 2*time.Second, "topic is created", func() bool {
|
||||
return helper.TopicClient.CheckTopic(reverseTopicName)
|
||||
})
|
||||
assert.True(t, created)
|
||||
})
|
||||
t.Run("notification should be configured after creating the topic", func(t *testing.T) {
|
||||
// check whether bucket notification added, should pass since topic got created
|
||||
notificationPresent := utils.Retry(12, 2*time.Second, "notification is created for bucket", func() bool {
|
||||
return helper.BucketClient.CheckBucketNotificationSetonRGW(namespace, storeName, reverseOBCName, reverseBucketName, reverseNotificationName, helper, objectStore.Spec.IsTLSEnabled())
|
||||
})
|
||||
assert.True(t, notificationPresent)
|
||||
})
|
||||
t.Run("delete CephBucketNotification: reverse-notification", func(t *testing.T) {
|
||||
err = helper.NotificationClient.DeleteNotification(reverseNotificationName, reverseTopicName)
|
||||
assert.Nil(t, err)
|
||||
t.Run("check notification removed from backend Bucket", func(t *testing.T) {
|
||||
// TODO: need to add that support
|
||||
t.Skipped()
|
||||
})
|
||||
})
|
||||
t.Run("delete CephBucketTopic: reverse-topic", func(t *testing.T) {
|
||||
err = helper.TopicClient.DeleteTopic(reverseTopicName, storeName, httpEndpointService)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
t.Run("delete ObjectBucketClaim: reverse-obc", func(t *testing.T) {
|
||||
err = helper.BucketClient.DeleteObc(reverseOBCName, bucketStorageClassName, reverseBucketName, maxObject, true)
|
||||
assert.Nil(t, err)
|
||||
logger.Info("Checking to see if the obc, secret, and cm have all been deleted")
|
||||
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(reverseOBCName, "deleted"); i++ {
|
||||
logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.NotEqual(t, 4, i)
|
||||
logger.Info("ensure OBC bucket was deleted")
|
||||
var rgwErr int
|
||||
for i = 0; i < 4; i++ {
|
||||
_, rgwErr, _ = rgw.GetBucket(rgwcontext, reverseBucketName)
|
||||
if rgwErr == rgw.RGWErrorNotFound {
|
||||
break
|
||||
}
|
||||
logger.Infof("(%d) check bucket deleted, sleeping for 5 seconds ...", i)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.NotEqual(t, 4, i)
|
||||
assert.Equal(t, rgwErr, rgw.RGWErrorNotFound)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("delete ObjectBucketClaim", func(t *testing.T) {
|
||||
i := 0
|
||||
dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
|
||||
assert.Nil(t, dobcErr)
|
||||
logger.Info("Checking to see if the obc, secret, and cm have all been deleted")
|
||||
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ {
|
||||
logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.NotEqual(t, 4, i)
|
||||
|
||||
logger.Info("ensure OBC bucket was deleted")
|
||||
var rgwErr int
|
||||
for i = 0; i < 4; i++ {
|
||||
_, rgwErr, _ = rgw.GetBucket(rgwcontext, bucketname)
|
||||
if rgwErr == rgw.RGWErrorNotFound {
|
||||
break
|
||||
}
|
||||
logger.Infof("(%d) check bucket deleted, sleeping for 5 seconds ...", i)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.NotEqual(t, 4, i)
|
||||
assert.Equal(t, rgwErr, rgw.RGWErrorNotFound)
|
||||
|
||||
dobErr := helper.BucketClient.DeleteBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete")
|
||||
assert.Nil(t, dobErr)
|
||||
})
|
||||
|
||||
t.Run("delete CephBucketNotification", func(t *testing.T) {
|
||||
err := helper.NotificationClient.DeleteNotification(notificationName, topicName)
|
||||
assert.Nil(t, err)
|
||||
t.Run("check notification removed from backend Bucket", func(t *testing.T) {
|
||||
// TODO: need to add that support
|
||||
t.Skipped()
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("delete CephBucketTopic", func(t *testing.T) {
|
||||
err := helper.TopicClient.DeleteTopic(topicName, storeName, httpEndpointService)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("delete CephObjectStore", func(t *testing.T) {
|
||||
deleteObjectStore(t, k8sh, namespace, storeName)
|
||||
assertObjectStoreDeletion(t, k8sh, namespace, storeName)
|
||||
})
|
||||
}
|
||||
@@ -46,6 +46,7 @@ import (
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner"
|
||||
"github.com/rook/rook/tests/integration/object/cosi"
|
||||
"github.com/rook/rook/tests/integration/object/notification"
|
||||
topickafka "github.com/rook/rook/tests/integration/object/topic/kafka"
|
||||
usercaps "github.com/rook/rook/tests/integration/object/user/caps"
|
||||
userkeys "github.com/rook/rook/tests/integration/object/user/keys"
|
||||
@@ -217,6 +218,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
|
||||
useropmask.Namespace,
|
||||
usercaps.Namespace,
|
||||
cosi.Namespace,
|
||||
notification.Namespace,
|
||||
)
|
||||
defer sharedObjectStore.Destroy()
|
||||
|
||||
@@ -228,10 +230,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
|
||||
// the ceph-cosi driver cannot reach a TLS object store endpoint, so this
|
||||
// suite skips itself in the TLS pass
|
||||
cosi.TestCephCOSIDriver(s.T(), k8sh, sharedObjectStore)
|
||||
|
||||
bucketNotificationTestStoreName := "bucket-notification-" + storeName
|
||||
createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable, swiftAndKeystone)
|
||||
testBucketNotifications(s, helper, k8sh, namespace, bucketNotificationTestStoreName)
|
||||
notification.TestBucketNotification(s.T(), k8sh, sharedObjectStore)
|
||||
}
|
||||
|
||||
func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, settings *installer.TestCephSettings, storeName string, swiftAndKeystone bool) {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
This tree holds the new-style integration tests for rook's object storage
|
||||
features. It is the model for converting the remaining old-style object tests
|
||||
(`testObjectStoreOperations`, store lifecycle, bucket notifications);
|
||||
(`testObjectStoreOperations`, store lifecycle);
|
||||
follow the conventions here exactly so the conversion happens once.
|
||||
|
||||
## Execution model
|
||||
@@ -34,11 +34,12 @@ must not collide with std-lib package names (no `io`, no `http`).
|
||||
|---|---|---|
|
||||
| `bucket/owner` | `object/bucket` | OBC `bucketOwner` handling |
|
||||
| `cosi` | `object/cosi` | CephCOSIDriver + COSI bucket provisioning |
|
||||
| `notification` | `object/notification` | CephBucketNotification HTTP endpoint delivery |
|
||||
| `topic/kafka` | `object/topic` | CephBucketTopic kafka endpoints |
|
||||
| `user/caps` | `object/user` | user capabilities |
|
||||
| `user/keys` | `object/user` | explicit S3 key management |
|
||||
| `user/opmask` | `object/user` | user op_mask |
|
||||
| reserved: `bucket/rw`, `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents`, `notification/*` | | future conversions |
|
||||
| reserved: `bucket/rw`, `bucket/quota`, `bucket/policy`, `bucket/lifecycle`, `tests/integration/object/lifecycle`, `tests/integration/object/dependents` | | future conversions |
|
||||
|
||||
Shared utilities live under `util/`:
|
||||
|
||||
@@ -211,5 +212,4 @@ verification; wire the dispatcher; delete the old code; retire
|
||||
| `testObjectStoreOperations` bucket lifecycle | `bucket/lifecycle` | per-OBC S3 client, move `lifecycleCmpOpts` out of the dispatcher |
|
||||
| `testObjectStoreOperations` deletion-blocked-by-dependents | `tests/integration/object/dependents` | private-store fixture, store condition predicates in `ready` |
|
||||
| `createCephObjectStore`/`runObjectE2ETestLite`/deletion asserts + zone.json canary | `tests/integration/object/lifecycle` | store create/health/delete helpers, `Sharedstore.Installer()` accessor |
|
||||
| `testBucketNotifications` | `notification/*` | HTTP endpoint fixture, per-OBC S3 client (the pod-log waiter is now `wait4.Assert/RequirePodLog`) |
|
||||
| upgrade-suite object usage | stays in upgrade suite | switch to typed clients + `wait4`/`ready` |
|
||||
@@ -0,0 +1,502 @@
|
||||
/*
|
||||
Copyright 2025 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package notification
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rgw "github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/rook/rook/tests/integration/object/util/client"
|
||||
"github.com/rook/rook/tests/integration/object/util/fixture"
|
||||
"github.com/rook/rook/tests/integration/object/util/sharedstore"
|
||||
"github.com/rook/rook/tests/integration/object/util/wait4"
|
||||
)
|
||||
|
||||
const sinkPort int32 = 8080
|
||||
|
||||
const (
|
||||
putEvent = "ObjectCreated:Put"
|
||||
deleteEvent = "ObjectRemoved:Delete"
|
||||
)
|
||||
|
||||
// httpTopic builds a CephBucketTopic that delivers to an HTTP endpoint.
|
||||
func httpTopic(name, namespace string, store *cephv1.CephObjectStore, uri string) *cephv1.CephBucketTopic {
|
||||
return &cephv1.CephBucketTopic{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
|
||||
Spec: cephv1.BucketTopicSpec{
|
||||
ObjectStoreName: store.Name,
|
||||
ObjectStoreNamespace: store.Namespace,
|
||||
Endpoint: cephv1.TopicEndpointSpec{HTTP: &cephv1.HTTPEndpointSpec{URI: uri}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// bucketNotification builds a CephBucketNotification for object put and delete events.
|
||||
func bucketNotification(name, namespace, topic string) *cephv1.CephBucketNotification {
|
||||
return &cephv1.CephBucketNotification{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
|
||||
Spec: cephv1.BucketNotificationSpec{
|
||||
Topic: topic,
|
||||
Events: []cephv1.BucketNotificationEvent{"s3:ObjectCreated:Put", "s3:ObjectRemoved:Delete"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// notificationLabel is the OBC label the operator watches to wire a notification to a bucket.
|
||||
func notificationLabel(notification string) string {
|
||||
return "bucket-notification-" + notification
|
||||
}
|
||||
|
||||
// notificationOBC builds an OBC labelled to subscribe its bucket to notification.
|
||||
func notificationOBC(name, namespace, storageClass, notification string) *bktv1alpha1.ObjectBucketClaim {
|
||||
return &bktv1alpha1.ObjectBucketClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Labels: map[string]string{notificationLabel(notification): notification},
|
||||
},
|
||||
Spec: bktv1alpha1.ObjectBucketClaimSpec{
|
||||
BucketName: name,
|
||||
StorageClassName: storageClass,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const Namespace = "test-notification"
|
||||
|
||||
func TestBucketNotification(t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore) {
|
||||
var (
|
||||
defaultName = Namespace
|
||||
objectStore = store.ObjectStore()
|
||||
|
||||
ns = &corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: defaultName,
|
||||
},
|
||||
}
|
||||
|
||||
sinkName = defaultName + "-sink"
|
||||
sinkSel = labels.SelectorFromSet(labels.Set{"app": sinkName})
|
||||
sinkURI = fmt.Sprintf("http://%s.%s:%d", sinkName, ns.Name, sinkPort)
|
||||
|
||||
storageClass = fixture.StorageClass(defaultName, objectStore)
|
||||
|
||||
topic = httpTopic(defaultName+"-topic", ns.Name, objectStore, sinkURI)
|
||||
notification = bucketNotification(defaultName+"-notification", ns.Name, topic.Name)
|
||||
obc = notificationOBC(defaultName+"-obc", ns.Name, storageClass.Name, notification.Name)
|
||||
|
||||
body = "test bucket notification payload"
|
||||
contentType = "text/plain"
|
||||
key1 = "obj1"
|
||||
key2 = "obj2"
|
||||
|
||||
btClient = k8sh.RookClientset.CephV1().CephBucketTopics(ns.Name)
|
||||
bnClient = k8sh.RookClientset.CephV1().CephBucketNotifications(ns.Name)
|
||||
obcClient = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name)
|
||||
)
|
||||
|
||||
t.Run("CephBucketNotification http", func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
|
||||
fixture.RequireNamespace(t, k8sh, ns)
|
||||
|
||||
requireHTTPSink(ctx, t, k8sh, ns.Name, sinkName)
|
||||
|
||||
fixture.RequireStorageClass(t, k8sh, storageClass)
|
||||
|
||||
t.Run(fmt.Sprintf("create CephBucketTopic %q", topic.Name), func(t *testing.T) {
|
||||
// topic creation may be slow right after rgw start up
|
||||
wait4.RequireCreate(ctx, t, btClient, topic, wait4.BucketTopic, wait4.TimeoutMedium)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("create CephBucketNotification %q", notification.Name), func(t *testing.T) {
|
||||
_, err := bnClient.Create(ctx, notification, metav1.CreateOptions{})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
s3agent := requireBoundOBC(ctx, t, k8sh, store, obc)
|
||||
|
||||
t.Run(fmt.Sprintf("notification %q is configured on bucket %q", notification.Name, obc.Spec.BucketName), func(t *testing.T) {
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("put object %q delivers a notification", key1), func(t *testing.T) {
|
||||
_, err := s3agent.PutObjectInBucket(ctx, obc.Spec.BucketName, body, key1, contentType)
|
||||
require.NoError(t, err)
|
||||
|
||||
requireNotificationReceived(ctx, t, k8sh, ns.Name, sinkSel, putEvent, key1)
|
||||
assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, deleteEvent, key1)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete object %q delivers a notification", key1), func(t *testing.T) {
|
||||
_, err := s3agent.DeleteObjectInBucket(ctx, obc.Spec.BucketName, key1)
|
||||
require.NoError(t, err)
|
||||
|
||||
requireNotificationReceived(ctx, t, k8sh, ns.Name, sinkSel, deleteEvent, key1)
|
||||
assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, putEvent, key1)
|
||||
})
|
||||
|
||||
t.Run("an unrelated OBC label change leaves the notification configured", func(t *testing.T) {
|
||||
t.Run(fmt.Sprintf("add a non-notification label to obc %q", obc.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) {
|
||||
l["test-label"] = "test-value"
|
||||
})
|
||||
})
|
||||
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true)
|
||||
|
||||
t.Run(fmt.Sprintf("remove the non-notification label from obc %q", obc.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) {
|
||||
delete(l, "test-label")
|
||||
})
|
||||
})
|
||||
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, true)
|
||||
})
|
||||
|
||||
t.Run("removing the notification label removes the notification from the bucket", func(t *testing.T) {
|
||||
t.Run(fmt.Sprintf("remove the notification label from obc %q", obc.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) {
|
||||
delete(l, notificationLabel(notification.Name))
|
||||
})
|
||||
})
|
||||
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, notification.Name, false)
|
||||
})
|
||||
|
||||
t.Run("an unsubscribed bucket delivers no notifications", func(t *testing.T) {
|
||||
t.Run(fmt.Sprintf("put object %q", key2), func(t *testing.T) {
|
||||
_, err := s3agent.PutObjectInBucket(ctx, obc.Spec.BucketName, body, key2, contentType)
|
||||
require.NoError(t, err)
|
||||
|
||||
assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, putEvent, key2)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete object %q", key2), func(t *testing.T) {
|
||||
_, err := s3agent.DeleteObjectInBucket(ctx, obc.Spec.BucketName, key2)
|
||||
require.NoError(t, err)
|
||||
|
||||
assertNotificationNotReceived(ctx, t, k8sh, ns.Name, sinkSel, deleteEvent, key2)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("a notification added to an existing OBC is configured on its bucket", func(t *testing.T) {
|
||||
addTopic := httpTopic(defaultName+"-topic-add", ns.Name, objectStore, sinkURI)
|
||||
addNotification := bucketNotification(defaultName+"-notification-add", ns.Name, addTopic.Name)
|
||||
|
||||
t.Run(fmt.Sprintf("create CephBucketTopic %q", addTopic.Name), func(t *testing.T) {
|
||||
wait4.RequireCreate(ctx, t, btClient, addTopic, wait4.BucketTopic, wait4.TimeoutMedium)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("create CephBucketNotification %q", addNotification.Name), func(t *testing.T) {
|
||||
_, err := bnClient.Create(ctx, addNotification, metav1.CreateOptions{})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("add the notification label for %q to obc %q", addNotification.Name, obc.Name), func(t *testing.T) {
|
||||
updateOBCLabels(ctx, t, k8sh, ns.Name, obc.Name, func(l map[string]string) {
|
||||
l[notificationLabel(addNotification.Name)] = addNotification.Name
|
||||
})
|
||||
})
|
||||
|
||||
requireBucketNotification(ctx, t, s3agent, obc.Spec.BucketName, addNotification.Name, true)
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketNotification %q", addNotification.Name), func(t *testing.T) {
|
||||
wait4.AssertDelete(ctx, t, bnClient, addNotification.Name, wait4.TimeoutShort)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketTopic %q", addTopic.Name), func(t *testing.T) {
|
||||
wait4.AssertDelete(ctx, t, btClient, addTopic.Name, wait4.TimeoutShort)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("a notification created before its topic is configured once the topic exists", func(t *testing.T) {
|
||||
revTopic := httpTopic(defaultName+"-topic-rev", ns.Name, objectStore, sinkURI)
|
||||
revNotification := bucketNotification(defaultName+"-notification-rev", ns.Name, revTopic.Name)
|
||||
revOBC := notificationOBC(defaultName+"-obc-rev", ns.Name, storageClass.Name, revNotification.Name)
|
||||
|
||||
revS3 := requireBoundOBC(ctx, t, k8sh, store, revOBC)
|
||||
|
||||
t.Run(fmt.Sprintf("create CephBucketNotification %q before its topic", revNotification.Name), func(t *testing.T) {
|
||||
_, err := bnClient.Create(ctx, revNotification, metav1.CreateOptions{})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("create CephBucketTopic %q", revTopic.Name), func(t *testing.T) {
|
||||
wait4.RequireCreate(ctx, t, btClient, revTopic, wait4.BucketTopic, wait4.TimeoutMedium)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("notification %q is configured on bucket %q once its topic exists", revNotification.Name, revOBC.Spec.BucketName), func(t *testing.T) {
|
||||
requireBucketNotification(ctx, t, revS3, revOBC.Spec.BucketName, revNotification.Name, true)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketNotification %q", revNotification.Name), func(t *testing.T) {
|
||||
wait4.AssertDelete(ctx, t, bnClient, revNotification.Name, wait4.TimeoutShort)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketTopic %q", revTopic.Name), func(t *testing.T) {
|
||||
wait4.AssertDelete(ctx, t, btClient, revTopic.Name, wait4.TimeoutShort)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete obc %q", revOBC.Name), func(t *testing.T) {
|
||||
deleteOBCAndBucket(ctx, t, k8sh, ns.Name, revOBC.Name)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete obc %q", obc.Name), func(t *testing.T) {
|
||||
deleteOBCAndBucket(ctx, t, k8sh, ns.Name, obc.Name)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketNotification %q", notification.Name), func(t *testing.T) {
|
||||
wait4.AssertDelete(ctx, t, bnClient, notification.Name, wait4.TimeoutShort)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("delete CephBucketTopic %q", topic.Name), func(t *testing.T) {
|
||||
wait4.AssertDelete(ctx, t, btClient, topic.Name, wait4.TimeoutShort)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) {
|
||||
list, err := obcClient.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, list.Items, 0)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("no CephBucketNotification(s) in ns %q", ns.Name), func(t *testing.T) {
|
||||
list, err := bnClient.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, list.Items, 0)
|
||||
})
|
||||
|
||||
t.Run(fmt.Sprintf("no CephBucketTopic(s) in ns %q", ns.Name), func(t *testing.T) {
|
||||
list, err := btClient.List(ctx, metav1.ListOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, list.Items, 0)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// requireBoundOBC creates obc, waits for it and its backing ObjectBucket to bind,
|
||||
// and returns an S3 client for the provisioned bucket.
|
||||
func requireBoundOBC(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore, obc *bktv1alpha1.ObjectBucketClaim) *rgw.S3Agent {
|
||||
t.Helper()
|
||||
|
||||
obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc.Namespace)
|
||||
obClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
|
||||
|
||||
var agent *rgw.S3Agent
|
||||
t.Run(fmt.Sprintf("create obc %q", obc.Name), func(t *testing.T) {
|
||||
liveOBC := wait4.RequireCreate(ctx, t, obcClient, obc, wait4.OBCBound, wait4.TimeoutLong)
|
||||
wait4.RequireCondition(ctx, t, obClient, liveOBC.Spec.ObjectBucketName, wait4.OBBound, wait4.TimeoutShort)
|
||||
|
||||
agent = obcS3Agent(ctx, t, k8sh, store, obc.Namespace, obc.Name)
|
||||
})
|
||||
return agent
|
||||
}
|
||||
|
||||
// deleteOBCAndBucket deletes the OBC and waits for its backing ObjectBucket (and
|
||||
// thus the rgw bucket) to be garbage-collected by the provisioner.
|
||||
func deleteOBCAndBucket(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string) {
|
||||
t.Helper()
|
||||
|
||||
obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace)
|
||||
obClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets()
|
||||
|
||||
liveOBC, err := obcClient.Get(ctx, name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
obName := liveOBC.Spec.ObjectBucketName
|
||||
|
||||
wait4.AssertDelete(ctx, t, obcClient, name, wait4.TimeoutShort)
|
||||
wait4.AssertAbsent(ctx, t, obClient, obName, wait4.TimeoutShort)
|
||||
}
|
||||
|
||||
// updateOBCLabels applies mutate to a live OBC's labels and updates it.
|
||||
func updateOBCLabels(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string, mutate func(map[string]string)) {
|
||||
t.Helper()
|
||||
|
||||
obcClient := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace)
|
||||
|
||||
liveOBC, err := obcClient.Get(ctx, name, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
if liveOBC.Labels == nil {
|
||||
liveOBC.Labels = map[string]string{}
|
||||
}
|
||||
mutate(liveOBC.Labels)
|
||||
|
||||
_, err = obcClient.Update(ctx, liveOBC, metav1.UpdateOptions{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// requireHTTPSink deploys a sample HTTP server that records the bucket
|
||||
// notifications rgw pushes to it, and waits for it to be ready. It needs no
|
||||
// explicit teardown: deleting the test namespace removes it.
|
||||
func requireHTTPSink(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace, name string) {
|
||||
t.Helper()
|
||||
|
||||
podLabels := map[string]string{"app": name}
|
||||
|
||||
deployment := &appsv1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
|
||||
Spec: appsv1.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{MatchLabels: podLabels},
|
||||
Template: corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: podLabels},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{{
|
||||
Name: name,
|
||||
// TODO: third party image, see https://github.com/rook/rook/issues/9741
|
||||
Image: "quay.io/jthottan/pythonwebserver:latest",
|
||||
Ports: []corev1.ContainerPort{{ContainerPort: sinkPort}},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
service := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Selector: podLabels,
|
||||
Ports: []corev1.ServicePort{{
|
||||
Port: sinkPort,
|
||||
TargetPort: intstr.FromInt32(sinkPort),
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
t.Run(fmt.Sprintf("create notification sink %q", name), func(t *testing.T) {
|
||||
wait4.RequireCreate(ctx, t, k8sh.Clientset.AppsV1().Deployments(namespace), deployment,
|
||||
func(d *appsv1.Deployment) bool { return d.Status.ReadyReplicas >= 1 }, wait4.TimeoutLong)
|
||||
wait4.RequireCreate(ctx, t, k8sh.Clientset.CoreV1().Services(namespace), service, nil, wait4.TimeoutShort)
|
||||
})
|
||||
}
|
||||
|
||||
// obcS3Agent builds an S3 client for the bucket provisioned by obcName, using the
|
||||
// credentials in the OBC's secret and the shared store's endpoint.
|
||||
func obcS3Agent(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, store *sharedstore.Sharedstore, namespace, obcName string) *rgw.S3Agent {
|
||||
t.Helper()
|
||||
|
||||
secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, obcName, metav1.GetOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
endpoint, err := client.GetS3Endpoint(store.ObjectStore(), k8sh, store.TLSEnabled())
|
||||
require.NoError(t, err)
|
||||
|
||||
agent, err := rgw.NewS3Agent(
|
||||
string(secret.Data["AWS_ACCESS_KEY_ID"]),
|
||||
string(secret.Data["AWS_SECRET_ACCESS_KEY"]),
|
||||
endpoint, true, nil, store.TLSEnabled(), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
return agent
|
||||
}
|
||||
|
||||
// requireBucketNotification waits until notificationName is (want=true) or is not
|
||||
// (want=false) configured on the rgw bucket, read via the S3 API.
|
||||
func requireBucketNotification(ctx context.Context, t *testing.T, agent *rgw.S3Agent, bucket, notificationName string, want bool) {
|
||||
t.Helper()
|
||||
|
||||
desc := fmt.Sprintf("notification %q configured=%v on bucket %q", notificationName, want, bucket)
|
||||
wait4.RequireEventually(ctx, t, wait4.TimeoutMedium, desc, func(ctx context.Context) error {
|
||||
conf, err := agent.Client.GetBucketNotificationConfiguration(ctx, &s3.GetBucketNotificationConfigurationInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, tc := range conf.TopicConfigurations {
|
||||
if tc.Id != nil && *tc.Id == notificationName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found != want {
|
||||
return fmt.Errorf("notification %q configured=%v, want %v", notificationName, found, want)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// requireNotificationReceived blocks until the sink logs a line for the given
|
||||
// event on the given object key.
|
||||
func requireNotificationReceived(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, sel labels.Selector, event, key string) {
|
||||
t.Helper()
|
||||
|
||||
wait4.RequirePodLog(ctx, t, k8sh, namespace, sel, wait4.TimeoutShort,
|
||||
func(line string) bool { return strings.Contains(line, event) && strings.Contains(line, key) },
|
||||
"notification sink received %s for %q", event, key)
|
||||
}
|
||||
|
||||
// notificationSettleDelay gives an unwanted notification time to arrive (if it
|
||||
// were going to) before we assert its absence.
|
||||
const notificationSettleDelay = 5 * time.Second
|
||||
|
||||
// assertNotificationNotReceived asserts the sink did not recently log the given
|
||||
// event for the given key. It matches only the most recent log lines: an earlier,
|
||||
// legitimately-received event for the same key may still be in the full history.
|
||||
func assertNotificationNotReceived(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, sel labels.Selector, event, key string) {
|
||||
t.Helper()
|
||||
|
||||
time.Sleep(notificationSettleDelay)
|
||||
|
||||
recent := tailSinkLog(ctx, t, k8sh, namespace, sel)
|
||||
assert.False(t, strings.Contains(recent, event) && strings.Contains(recent, key),
|
||||
"notification sink unexpectedly received %s for %q", event, key)
|
||||
}
|
||||
|
||||
// sinkLogTailLines bounds assertNotificationNotReceived to recent log lines.
|
||||
const sinkLogTailLines = 5
|
||||
|
||||
// tailSinkLog returns the last sinkLogTailLines log lines of the first pod
|
||||
// matching sel.
|
||||
func tailSinkLog(ctx context.Context, t *testing.T, k8sh *utils.K8sHelper, namespace string, sel labels.Selector) string {
|
||||
t.Helper()
|
||||
|
||||
pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: sel.String()})
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, pods.Items)
|
||||
|
||||
tail := int64(sinkLogTailLines)
|
||||
stream, err := k8sh.Clientset.CoreV1().Pods(namespace).
|
||||
GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{TailLines: &tail}).
|
||||
Stream(ctx)
|
||||
require.NoError(t, err)
|
||||
defer stream.Close()
|
||||
|
||||
out, err := io.ReadAll(stream)
|
||||
require.NoError(t, err)
|
||||
return string(out)
|
||||
}
|
||||
Reference in new issue
Block a user