diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go index f20599531..1f9f5660d 100644 --- a/tests/integration/ceph_object_test.go +++ b/tests/integration/ceph_object_test.go @@ -40,9 +40,10 @@ import ( "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" - "github.com/rook/rook/tests/integration/object/bucketowner" + bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner" topickafka "github.com/rook/rook/tests/integration/object/topic/kafka" - "github.com/rook/rook/tests/integration/object/user/userkeys" + userkeys "github.com/rook/rook/tests/integration/object/user/keys" + "github.com/rook/rook/tests/integration/object/util/sharedstore" ) const ( @@ -156,9 +157,18 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install // now test operation of the first object store testObjectStoreOperations(s, helper, k8sh, settings, storeName, swiftAndKeystone) - bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, installer, logger, tlsEnable) - userkeys.TestObjectStoreUserKeys(s.T(), k8sh, installer, logger, tlsEnable) - topickafka.TestBucketTopicKafka(s.T(), k8sh, installer, logger, tlsEnable) + // The namespaced test packages below all skip when TLS is enabled, so only set up the + // shared store when it will actually be used. + var sharedObjectStore *cephv1.CephObjectStore + if !tlsEnable { + var teardownSharedStore func() + sharedObjectStore, teardownSharedStore = sharedstore.Setup(s.T(), k8sh) + defer teardownSharedStore() + + bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore) + userkeys.TestObjectStoreUserKeys(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore) + topickafka.TestBucketTopicKafka(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore) + } bucketNotificationTestStoreName := "bucket-notification-" + storeName createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable, swiftAndKeystone) diff --git a/tests/integration/object/bucketowner/bucketowner.go b/tests/integration/object/bucket/owner/owner.go similarity index 83% rename from tests/integration/object/bucketowner/bucketowner.go rename to tests/integration/object/bucket/owner/owner.go index a1097463f..5c3243c57 100644 --- a/tests/integration/object/bucketowner/bucketowner.go +++ b/tests/integration/object/bucket/owner/owner.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package bucketowner +package owner import ( "bufio" @@ -39,150 +39,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/intstr" -) - -var ( - defaultName = "test-bucket-owner" - - ns = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName, - }, - } - - objectStore = &cephv1.CephObjectStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName, - // the CephObjectstore must be in the same ns as the CephCluster - Namespace: "object-ns", - }, - Spec: cephv1.ObjectStoreSpec{ - MetadataPool: cephv1.PoolSpec{ - Replicated: cephv1.ReplicatedSpec{ - Size: 1, - RequireSafeReplicaSize: false, - }, - }, - DataPool: cephv1.PoolSpec{ - Replicated: cephv1.ReplicatedSpec{ - Size: 1, - RequireSafeReplicaSize: false, - }, - }, - Gateway: cephv1.GatewaySpec{ - Port: 80, - Instances: 1, - }, - AllowUsersInNamespaces: []string{ns.Name}, - }, - } - - objectStoreSvc = &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: objectStore.Name, - Namespace: objectStore.Namespace, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": "rook-ceph-rgw", - "rook_cluster": objectStore.Namespace, - "rook_object_store": objectStore.Name, - }, - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: 80, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt(8080), - }, - }, - SessionAffinity: corev1.ServiceAffinityNone, - Type: corev1.ServiceTypeNodePort, - }, - } - - storageClass = &storagev1.StorageClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName, - }, - Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket", - Parameters: map[string]string{ - "objectStoreName": objectStore.Name, - "objectStoreNamespace": objectStore.Namespace, - }, - } - - // test user without any quotas set - osu1 = cephv1.CephObjectStoreUser{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-user1", - Namespace: ns.Name, - }, - Spec: cephv1.ObjectStoreUserSpec{ - Store: objectStore.Name, - ClusterNamespace: objectStore.Namespace, - }, - } - - // test user with quotas set - osu2 = cephv1.CephObjectStoreUser{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-user2", - Namespace: ns.Name, - }, - Spec: cephv1.ObjectStoreUserSpec{ - Store: objectStore.Name, - ClusterNamespace: objectStore.Namespace, - Quotas: &cephv1.ObjectUserQuotaSpec{ - MaxBuckets: func(i int) *int { return &i }(1111), - MaxSize: resource.NewQuantity(2222, resource.DecimalSI), - MaxObjects: func(i int64) *int64 { return &i }(3333), - }, - }, - } - - obc1 = bktv1alpha1.ObjectBucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-obc1", - Namespace: ns.Name, - }, - Spec: bktv1alpha1.ObjectBucketClaimSpec{ - BucketName: defaultName + "-obc1", - StorageClassName: storageClass.Name, - AdditionalConfig: map[string]string{ - "bucketOwner": osu1.Name, - }, - }, - } - - obc2 = bktv1alpha1.ObjectBucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-obc2", - Namespace: ns.Name, - }, - Spec: bktv1alpha1.ObjectBucketClaimSpec{ - BucketName: defaultName + "-obc2", - StorageClassName: storageClass.Name, - AdditionalConfig: map[string]string{ - "bucketOwner": osu1.Name, - }, - }, - } - - obcBogusOwner = bktv1alpha1.ObjectBucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-bogus-owner", - Namespace: ns.Name, - }, - Spec: bktv1alpha1.ObjectBucketClaimSpec{ - BucketName: defaultName, - StorageClassName: storageClass.Name, - AdditionalConfig: map[string]string{ - "bucketOwner": defaultName + "-bogus-user", - }, - }, - } ) func WaitForPodLogContainingText(k8sh *utils.K8sHelper, namespace string, selector *labels.Selector, text string, timeout time.Duration) error { @@ -231,7 +87,98 @@ func WaitForPodLogContainingText(k8sh *utils.K8sHelper, namespace string, select return nil } -func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool) { +func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) { + var ( + defaultName = "test-bucketowner" + + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + } + + storageClass = &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket", + Parameters: map[string]string{ + "objectStoreName": objectStore.Name, + "objectStoreNamespace": objectStore.Namespace, + }, + } + + // test user without any quotas set + osu1 = cephv1.CephObjectStoreUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-user1", + Namespace: ns.Name, + }, + Spec: cephv1.ObjectStoreUserSpec{ + Store: objectStore.Name, + ClusterNamespace: objectStore.Namespace, + }, + } + + // test user with quotas set + osu2 = cephv1.CephObjectStoreUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-user2", + Namespace: ns.Name, + }, + Spec: cephv1.ObjectStoreUserSpec{ + Store: objectStore.Name, + ClusterNamespace: objectStore.Namespace, + Quotas: &cephv1.ObjectUserQuotaSpec{ + MaxBuckets: func(i int) *int { return &i }(1111), + MaxSize: resource.NewQuantity(2222, resource.DecimalSI), + MaxObjects: func(i int64) *int64 { return &i }(3333), + }, + }, + } + + obc1 = bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-obc1", + Namespace: ns.Name, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: defaultName + "-obc1", + StorageClassName: storageClass.Name, + AdditionalConfig: map[string]string{ + "bucketOwner": osu1.Name, + }, + }, + } + + obc2 = bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-obc2", + Namespace: ns.Name, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: defaultName + "-obc2", + StorageClassName: storageClass.Name, + AdditionalConfig: map[string]string{ + "bucketOwner": osu1.Name, + }, + }, + } + + obcBogusOwner = bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-bogus-owner", + Namespace: ns.Name, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: defaultName, + StorageClassName: storageClass.Name, + AdditionalConfig: map[string]string{ + "bucketOwner": defaultName + "-bogus-user", + }, + }, + } + ) t.Run("OBC bucketOwner", func(t *testing.T) { if tlsEnable { // There is lots of coverage of rgw working with tls enabled; skipping to save time. @@ -249,31 +196,6 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta require.NoError(t, err) }) - t.Run(fmt.Sprintf("create CephObjectStore %q", objectStore.Name), func(t *testing.T) { - objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{}) - require.NoError(t, err) - - osReady := utils.Retry(180, time.Second, "CephObjectStore is Ready", func() bool { - liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{}) - if err != nil { - return false - } - - if liveOs.Status == nil { - return false - } - - // return liveOs.Status.Phase == "Ready" - return liveOs.Status.Phase == cephv1.ConditionReady - }) - require.True(t, osReady) - }) - - t.Run(fmt.Sprintf("create svc %q", objectStoreSvc.Name), func(t *testing.T) { - _, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, objectStoreSvc, metav1.CreateOptions{}) - require.NoError(t, err) - }) - t.Run(fmt.Sprintf("create sc %q", storageClass.Name), func(t *testing.T) { _, err := k8sh.Clientset.StorageV1().StorageClasses().Create(ctx, storageClass, metav1.CreateOptions{}) require.NoError(t, err) @@ -783,16 +705,6 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta require.NoError(t, err) }) - t.Run(fmt.Sprintf("delete svc %q", objectStoreSvc.Name), func(t *testing.T) { - err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStoreSvc.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }) - - t.Run(fmt.Sprintf("delete CephObjectStore %q", objectStore.Name), func(t *testing.T) { - err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }) - t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) { err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}) require.NoError(t, err) diff --git a/tests/integration/object/topic/kafka/kafka.go b/tests/integration/object/topic/kafka/kafka.go index 3cb23bd24..f7e556c88 100644 --- a/tests/integration/object/topic/kafka/kafka.go +++ b/tests/integration/object/topic/kafka/kafka.go @@ -32,7 +32,6 @@ import ( corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/tests/framework/installer" @@ -40,176 +39,6 @@ import ( utilsns "github.com/rook/rook/tests/integration/object/util/sns" ) -var ( - defaultName = "test-topickafka" - - ns = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName, - }, - } - - objectStore = &cephv1.CephObjectStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName, - // the CephObjectstore must be in the same ns as the CephCluster - Namespace: "object-ns", - }, - Spec: cephv1.ObjectStoreSpec{ - MetadataPool: cephv1.PoolSpec{ - Replicated: cephv1.ReplicatedSpec{ - Size: 1, - RequireSafeReplicaSize: false, - }, - }, - DataPool: cephv1.PoolSpec{ - Replicated: cephv1.ReplicatedSpec{ - Size: 1, - RequireSafeReplicaSize: false, - }, - }, - Gateway: cephv1.GatewaySpec{ - Port: 80, - Instances: 1, - }, - AllowUsersInNamespaces: []string{ns.Name}, - }, - } - - objectStoreSvc = &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: objectStore.Name, - Namespace: objectStore.Namespace, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": "rook-ceph-rgw", - "rook_cluster": objectStore.Namespace, - "rook_object_store": objectStore.Name, - }, - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: 80, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt(8080), - }, - }, - SessionAffinity: corev1.ServiceAffinityNone, - Type: corev1.ServiceTypeNodePort, - }, - } - - secret1 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret1", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "user-name": []byte("kafka-user1"), - }, - } - - secret2 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret2", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "password": []byte("kafka-pass2"), - }, - } - - secret3 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret3", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "user-name": []byte("kafka-user3"), - }, - } - - secret4 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret4", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "password": []byte("kafka-pass4"), - }, - } - - storageClass = &storagev1.StorageClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName, - }, - Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket", - Parameters: map[string]string{ - "objectStoreName": objectStore.Name, - "objectStoreNamespace": objectStore.Namespace, - }, - } - - obc1 = &bktv1alpha1.ObjectBucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-obc1", - Namespace: ns.Name, - Labels: map[string]string{ - "bucket-notification-" + bn1.Name: bn1.Name, - }, - }, - Spec: bktv1alpha1.ObjectBucketClaimSpec{ - BucketName: defaultName + "-obc1", - StorageClassName: storageClass.Name, - }, - } - - bt1 = &cephv1.CephBucketTopic{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-topic1", - Namespace: ns.Name, - }, - Spec: cephv1.BucketTopicSpec{ - ObjectStoreName: objectStore.Name, - ObjectStoreNamespace: objectStore.Namespace, - Persistent: false, - Endpoint: cephv1.TopicEndpointSpec{ - Kafka: &cephv1.KafkaEndpointSpec{ - URI: "kafka://kafka.example.com:9094", - AckLevel: "broker", - UseSSL: false, - UserSecretRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret1.Name, - }, - Key: "user-name", - }, - PasswordSecretRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret2.Name, - }, - Key: "password", - }, - }, - }, - }, - } - - bn1 = &cephv1.CephBucketNotification{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-notification1", - Namespace: ns.Name, - }, - Spec: cephv1.BucketNotificationSpec{ - Topic: bt1.Name, - Events: []cephv1.BucketNotificationEvent{ - "s3:ObjectCreated:*", - }, - }, - } -) - func checkStatusSecrets(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBucketTopic, expectedSecrets []*corev1.Secret) { t.Run(fmt.Sprintf("cephBucketTopic %q has .status.secrets set", bt.Name), func(t *testing.T) { ctx := context.TODO() @@ -301,9 +130,128 @@ func cephBucketTopicReady(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBu return liveBt } -func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool) { - ctx := context.TODO() - var snsClient *sns.Client +func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) { + var ( + defaultName = "test-topickafka" + + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + } + + secret1 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret1", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "user-name": []byte("kafka-user1"), + }, + } + + secret2 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret2", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "password": []byte("kafka-pass2"), + }, + } + + secret3 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret3", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "user-name": []byte("kafka-user3"), + }, + } + + secret4 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret4", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "password": []byte("kafka-pass4"), + }, + } + + storageClass = &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket", + Parameters: map[string]string{ + "objectStoreName": objectStore.Name, + "objectStoreNamespace": objectStore.Namespace, + }, + } + + bt1 = &cephv1.CephBucketTopic{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-topic1", + Namespace: ns.Name, + }, + Spec: cephv1.BucketTopicSpec{ + ObjectStoreName: objectStore.Name, + ObjectStoreNamespace: objectStore.Namespace, + Persistent: false, + Endpoint: cephv1.TopicEndpointSpec{ + Kafka: &cephv1.KafkaEndpointSpec{ + URI: "kafka://kafka.example.com:9094", + AckLevel: "broker", + UseSSL: false, + UserSecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret1.Name, + }, + Key: "user-name", + }, + PasswordSecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret2.Name, + }, + Key: "password", + }, + }, + }, + }, + } + + bn1 = &cephv1.CephBucketNotification{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-notification1", + Namespace: ns.Name, + }, + Spec: cephv1.BucketNotificationSpec{ + Topic: bt1.Name, + Events: []cephv1.BucketNotificationEvent{ + "s3:ObjectCreated:*", + }, + }, + } + + obc1 = &bktv1alpha1.ObjectBucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-obc1", + Namespace: ns.Name, + Labels: map[string]string{ + "bucket-notification-" + bn1.Name: bn1.Name, + }, + }, + Spec: bktv1alpha1.ObjectBucketClaimSpec{ + BucketName: defaultName + "-obc1", + StorageClassName: storageClass.Name, + }, + } + + ctx = context.TODO() + snsClient *sns.Client + ) t.Run("CephBucketTopic kafka", func(t *testing.T) { if tlsEnable { @@ -316,30 +264,6 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal require.NoError(t, err) }) - t.Run(fmt.Sprintf("create CephObjectStore %q", objectStore.Name), func(t *testing.T) { - objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{}) - require.NoError(t, err) - - osReady := utils.Retry(180, time.Second, "CephObjectStore is Ready", func() bool { - liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{}) - if err != nil { - return false - } - - if liveOs.Status == nil { - return false - } - - return liveOs.Status.Phase == cephv1.ConditionReady - }) - require.True(t, osReady) - }) - - t.Run(fmt.Sprintf("create svc %q", objectStoreSvc.Name), func(t *testing.T) { - _, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, objectStoreSvc, metav1.CreateOptions{}) - require.NoError(t, err) - }) - t.Run(fmt.Sprintf("create secret %q", secret1.Name), func(t *testing.T) { _, err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Create(ctx, secret1, metav1.CreateOptions{}) require.NoError(t, err) @@ -386,7 +310,7 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal t.Run("setup sns client", func(t *testing.T) { var err error - snsClient, err = utilsns.NewClient(objectStore, objectStoreSvc, k8sh, installer, tlsEnable) + snsClient, err = utilsns.NewClient(objectStore, k8sh, installer, tlsEnable) require.NoError(t, err) }) @@ -683,16 +607,6 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal assert.Len(t, secrets.Items, 0) }) - t.Run(fmt.Sprintf("delete svc %q", objectStoreSvc.Name), func(t *testing.T) { - err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStoreSvc.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }) - - t.Run(fmt.Sprintf("delete CephObjectStore %q", objectStore.Name), func(t *testing.T) { - err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }) - t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) { err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}) require.NoError(t, err) diff --git a/tests/integration/object/user/userkeys/userkeys.go b/tests/integration/object/user/keys/keys.go similarity index 73% rename from tests/integration/object/user/userkeys/userkeys.go rename to tests/integration/object/user/keys/keys.go index f7458cd81..9cefcdba7 100644 --- a/tests/integration/object/user/userkeys/userkeys.go +++ b/tests/integration/object/user/keys/keys.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package userkeys +package keys import ( "context" @@ -28,7 +28,6 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/tests/framework/installer" @@ -36,180 +35,9 @@ import ( utiladmin "github.com/rook/rook/tests/integration/object/util/admin" ) -var ( - defaultName = "test-userkeys" - - ns = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName, - }, - } - - objectStore = &cephv1.CephObjectStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName, - // the CephObjectstore must be in the same ns as the CephCluster - Namespace: "object-ns", - }, - Spec: cephv1.ObjectStoreSpec{ - MetadataPool: cephv1.PoolSpec{ - Replicated: cephv1.ReplicatedSpec{ - Size: 1, - RequireSafeReplicaSize: false, - }, - }, - DataPool: cephv1.PoolSpec{ - Replicated: cephv1.ReplicatedSpec{ - Size: 1, - RequireSafeReplicaSize: false, - }, - }, - Gateway: cephv1.GatewaySpec{ - Port: 80, - Instances: 1, - }, - AllowUsersInNamespaces: []string{ns.Name}, - }, - } - - objectStoreSvc = &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: objectStore.Name, - Namespace: objectStore.Namespace, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": "rook-ceph-rgw", - "rook_cluster": objectStore.Namespace, - "rook_object_store": objectStore.Name, - }, - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: 80, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt(8080), - }, - }, - SessionAffinity: corev1.ServiceAffinityNone, - Type: corev1.ServiceTypeNodePort, - }, - } - - secret1 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret1", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "AWS_ACCESS_KEY_ID": []byte("P1ANF2BP4K2LPGOR5SBB"), - "AWS_SECRET_ACCESS_KEY": []byte("yfCHPhhOed0vJkqZsGEODyJrdKqHD09OMTWCnwjX"), - }, - } - - secret2 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret2", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "AWS_ACCESS_KEY_ID": []byte("2I6RPUTQFMNCSYEXZ6VM"), - "AWS_SECRET_ACCESS_KEY": []byte("uY066SWPfaOVlDeYc7GJyOTfkDejyDdXrqehS6wx"), - }, - } - - secret3 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret3", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "AWS_ACCESS_KEY_ID": []byte("J4D0P20F3EDR51OSND7Y"), - "AWS_SECRET_ACCESS_KEY": []byte("jn89OpkXNoDlIHVVQ23mE2DZgPmuDK3WVH5ExOvQ"), - }, - } - - secret4 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret4", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "AWS_ACCESS_KEY_ID": []byte("MPZX7DG5WJWQ6VPCSLYT"), - "AWS_SECRET_ACCESS_KEY": []byte("phh7DIxnLPeSD2V6FUouhmnWrKlKRD5dBykyXozX"), - }, - } - - secret5 = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-secret5", - Namespace: ns.Name, - }, - Data: map[string][]byte{ - "AWS_ACCESS_KEY_ID": []byte("7TNSSANCO5KXK23IPT91"), - "AWS_SECRET_ACCESS_KEY": []byte("HksEDf0hEh3PtTvl7s9x6CyXfkWuY8eAMYAcvH5l"), - }, - } - - osu1 = cephv1.CephObjectStoreUser{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultName + "-user1", - Namespace: ns.Name, - }, - Spec: cephv1.ObjectStoreUserSpec{ - Store: objectStore.Name, - ClusterNamespace: objectStore.Namespace, - Keys: []cephv1.ObjectUserKey{ - { - AccessKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret1.Name, - }, - Key: "AWS_ACCESS_KEY_ID", - }, - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret1.Name, - }, - Key: "AWS_SECRET_ACCESS_KEY", - }, - }, - { - AccessKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret2.Name, - }, - Key: "AWS_ACCESS_KEY_ID", - }, - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret2.Name, - }, - Key: "AWS_SECRET_ACCESS_KEY", - }, - }, - { - AccessKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret3.Name, - }, - Key: "AWS_ACCESS_KEY_ID", - }, - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret3.Name, - }, - Key: "AWS_SECRET_ACCESS_KEY", - }, - }, - }, - }, - } -) - // generate the secret name the operator is expected to generate for the CephObjectStoreUser func generateObjectStoreUserSecretName(osu cephv1.CephObjectStoreUser) string { - return "rook-ceph-object-user-" + osu.Namespace + "-" + osu.Name + return "rook-ceph-object-user-" + osu.Spec.Store + "-" + osu.Name } // find a UserKeySpec by AccessKey value @@ -226,7 +54,7 @@ func checkStatusKeys(t *testing.T, k8sh *utils.K8sHelper, osu cephv1.CephObjectS t.Run(fmt.Sprintf("cephObjectStoreUser %q has .status.keys set", osu.Name), func(t *testing.T) { ctx := context.TODO() - liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu.Name, metav1.GetOptions{}) + liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(osu.Namespace).Get(ctx, osu.Name, metav1.GetOptions{}) require.NoError(t, err) require.NotNil(t, liveOsu.Status) @@ -244,7 +72,7 @@ func checkStatusKeys(t *testing.T, k8sh *utils.K8sHelper, osu cephv1.CephObjectS require.NoError(t, err) // fetch the live secret for UID and ResourceVersion - liveSecret, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Get(ctx, secret.Name, metav1.GetOptions{}) + liveSecret, err := k8sh.Clientset.CoreV1().Secrets(osu.Namespace).Get(ctx, secret.Name, metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, liveSecret.Name, secretRef.Name) @@ -284,7 +112,126 @@ func checkRgwUserKeys(t *testing.T, adminClient *admin.API, osu cephv1.CephObjec }) } -func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool) { +func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) { + var ( + defaultName = "test-userkeys" + + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + } + + secret1 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret1", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("P1ANF2BP4K2LPGOR5SBB"), + "AWS_SECRET_ACCESS_KEY": []byte("yfCHPhhOed0vJkqZsGEODyJrdKqHD09OMTWCnwjX"), + }, + } + + secret2 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret2", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("2I6RPUTQFMNCSYEXZ6VM"), + "AWS_SECRET_ACCESS_KEY": []byte("uY066SWPfaOVlDeYc7GJyOTfkDejyDdXrqehS6wx"), + }, + } + + secret3 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret3", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("J4D0P20F3EDR51OSND7Y"), + "AWS_SECRET_ACCESS_KEY": []byte("jn89OpkXNoDlIHVVQ23mE2DZgPmuDK3WVH5ExOvQ"), + }, + } + + secret4 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret4", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("MPZX7DG5WJWQ6VPCSLYT"), + "AWS_SECRET_ACCESS_KEY": []byte("phh7DIxnLPeSD2V6FUouhmnWrKlKRD5dBykyXozX"), + }, + } + + secret5 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-secret5", + Namespace: ns.Name, + }, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("7TNSSANCO5KXK23IPT91"), + "AWS_SECRET_ACCESS_KEY": []byte("HksEDf0hEh3PtTvl7s9x6CyXfkWuY8eAMYAcvH5l"), + }, + } + + osu1 = cephv1.CephObjectStoreUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName + "-user1", + Namespace: ns.Name, + }, + Spec: cephv1.ObjectStoreUserSpec{ + Store: objectStore.Name, + ClusterNamespace: objectStore.Namespace, + Keys: []cephv1.ObjectUserKey{ + { + AccessKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret1.Name, + }, + Key: "AWS_ACCESS_KEY_ID", + }, + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret1.Name, + }, + Key: "AWS_SECRET_ACCESS_KEY", + }, + }, + { + AccessKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret2.Name, + }, + Key: "AWS_ACCESS_KEY_ID", + }, + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret2.Name, + }, + Key: "AWS_SECRET_ACCESS_KEY", + }, + }, + { + AccessKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret3.Name, + }, + Key: "AWS_ACCESS_KEY_ID", + }, + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret3.Name, + }, + Key: "AWS_SECRET_ACCESS_KEY", + }, + }, + }, + }, + } + ) t.Run("ObjectStoreUser keys", func(t *testing.T) { if tlsEnable { // Skip testing with and without TLS to reduce test time @@ -299,30 +246,6 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins require.NoError(t, err) }) - t.Run(fmt.Sprintf("create CephObjectStore %q", objectStore.Name), func(t *testing.T) { - objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{}) - require.NoError(t, err) - - osReady := utils.Retry(180, time.Second, "CephObjectStore is Ready", func() bool { - liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{}) - if err != nil { - return false - } - - if liveOs.Status == nil { - return false - } - - return liveOs.Status.Phase == cephv1.ConditionReady - }) - require.True(t, osReady) - }) - - t.Run(fmt.Sprintf("create svc %q", objectStoreSvc.Name), func(t *testing.T) { - _, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, objectStoreSvc, metav1.CreateOptions{}) - require.NoError(t, err) - }) - t.Run(fmt.Sprintf("create secret %q", secret1.Name), func(t *testing.T) { _, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Create(ctx, secret1, metav1.CreateOptions{}) require.NoError(t, err) @@ -665,16 +588,6 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins assert.Len(t, secrets.Items, 0) }) - t.Run(fmt.Sprintf("delete svc %q", objectStoreSvc.Name), func(t *testing.T) { - err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStoreSvc.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }) - - t.Run(fmt.Sprintf("delete CephObjectStore %q", objectStore.Name), func(t *testing.T) { - err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }) - t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) { err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}) require.NoError(t, err) diff --git a/tests/integration/object/util/sharedstore/sharedstore.go b/tests/integration/object/util/sharedstore/sharedstore.go new file mode 100644 index 000000000..bd9e112c8 --- /dev/null +++ b/tests/integration/object/util/sharedstore/sharedstore.go @@ -0,0 +1,129 @@ +/* +Copyright 2026 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package sharedstore provides a shared CephObjectStore fixture for +// tests under tests/integration/object. A single store is created once +// and torn down after all sub-package tests complete, avoiding the +// per-package setup/teardown overhead. +package sharedstore + +import ( + "context" + "testing" + "time" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// Setup creates a shared CephObjectStore and its NodePort Service in +// ObjectStoreNamespace, waits for the store to become Ready, and returns a +// teardown function that deletes both resources. The teardown function should +// be deferred by the caller. +func Setup(t *testing.T, k8sh *utils.K8sHelper) (*cephv1.CephObjectStore, func()) { + t.Helper() + ctx := context.TODO() + + objectStore := &cephv1.CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shared", + Namespace: "object-ns", + }, + Spec: cephv1.ObjectStoreSpec{ + MetadataPool: cephv1.PoolSpec{ + Replicated: cephv1.ReplicatedSpec{ + Size: 1, + RequireSafeReplicaSize: false, + }, + }, + DataPool: cephv1.PoolSpec{ + Replicated: cephv1.ReplicatedSpec{ + Size: 1, + RequireSafeReplicaSize: false, + }, + }, + Gateway: cephv1.GatewaySpec{ + Port: 80, + Instances: 1, + }, + // Allow CephObjectStoreUser resources from every namespace used by + // the sub-package tests under tests/integration/object. + AllowUsersInNamespaces: []string{ + "test-bucketowner", + "test-topickafka", + "test-userkeys", + "test-useropmask", + }, + }, + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + // The service name must match ObjectStoreName so that + // util/s3.GetS3Endpoint can locate it by objectStore.Name. + Name: objectStore.Name, + Namespace: objectStore.Namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": "rook-ceph-rgw", + "rook_cluster": objectStore.Namespace, + "rook_object_store": objectStore.Name, + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(8080), + }, + }, + SessionAffinity: corev1.ServiceAffinityNone, + Type: corev1.ServiceTypeNodePort, + }, + } + + _, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{}) + require.NoError(t, err) + + osReady := utils.Retry(180, time.Second, "shared CephObjectStore is Ready", func() bool { + liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{}) + if err != nil { + return false + } + if liveOs.Status == nil { + return false + } + return liveOs.Status.Phase == cephv1.ConditionReady + }) + require.True(t, osReady, "shared CephObjectStore did not become Ready") + + _, err = k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, svc, metav1.CreateOptions{}) + require.NoError(t, err) + + return objectStore, func() { + if err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}); err != nil { + t.Logf("failed to delete shared object store service: %v", err) + } + if err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}); err != nil { + t.Logf("failed to delete shared CephObjectStore: %v", err) + } + } +} diff --git a/tests/integration/object/util/sns/sns.go b/tests/integration/object/util/sns/sns.go index d537cd220..bddbdfdb5 100644 --- a/tests/integration/object/util/sns/sns.go +++ b/tests/integration/object/util/sns/sns.go @@ -25,7 +25,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sns" smithyendpoints "github.com/aws/smithy-go/endpoints" "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/tests/framework/installer" @@ -49,7 +48,7 @@ func (r *snsResolverV2) ResolveEndpoint(ctx context.Context, params sns.Endpoint }, nil } -func NewClient(objectStore *cephv1.CephObjectStore, svc *corev1.Service, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) { +func NewClient(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) { ctx := context.TODO() accessKey, secretKey, err := utils3.GetS3Credentials(objectStore, installer)