test: factor out shared object store creation from ns'd object suite tests

This factors out the creation of a single CephObjectStore instance to be
shared between the individually namespaced object suite packages.

This reduces the object suite runtime by the cost of starting up and
tearing down an object store three times.

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
(cherry picked from commit 92f6d3fb67)
This commit is contained in:
Joshua Hoblitt committed 2026-04-01 11:46:08 -07:00
1 parent 640b9ccea1
commit c0d210f260
6 files changed
+485 -608

No files matched your search

+15 -5
View File
@@ -40,9 +40,10 @@ import (
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/rook/rook/tests/integration/object/bucketowner"
bucketowner "github.com/rook/rook/tests/integration/object/bucket/owner"
topickafka "github.com/rook/rook/tests/integration/object/topic/kafka"
"github.com/rook/rook/tests/integration/object/user/userkeys"
userkeys "github.com/rook/rook/tests/integration/object/user/keys"
"github.com/rook/rook/tests/integration/object/util/sharedstore"
)
const (
@@ -156,9 +157,18 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
// now test operation of the first object store
testObjectStoreOperations(s, helper, k8sh, settings, storeName, swiftAndKeystone)
bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, installer, logger, tlsEnable)
userkeys.TestObjectStoreUserKeys(s.T(), k8sh, installer, logger, tlsEnable)
topickafka.TestBucketTopicKafka(s.T(), k8sh, installer, logger, tlsEnable)
// The namespaced test packages below all skip when TLS is enabled, so only set up the
// shared store when it will actually be used.
var sharedObjectStore *cephv1.CephObjectStore
if !tlsEnable {
var teardownSharedStore func()
sharedObjectStore, teardownSharedStore = sharedstore.Setup(s.T(), k8sh)
defer teardownSharedStore()
bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore)
userkeys.TestObjectStoreUserKeys(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore)
topickafka.TestBucketTopicKafka(s.T(), k8sh, installer, logger, tlsEnable, sharedObjectStore)
}
bucketNotificationTestStoreName := "bucket-notification-" + storeName
createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable, swiftAndKeystone)
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package bucketowner
package owner
import (
"bufio"
@@ -39,150 +39,6 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/intstr"
)
var (
defaultName = "test-bucket-owner"
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
}
objectStore = &cephv1.CephObjectStore{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
// the CephObjectstore must be in the same ns as the CephCluster
Namespace: "object-ns",
},
Spec: cephv1.ObjectStoreSpec{
MetadataPool: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{
Size: 1,
RequireSafeReplicaSize: false,
},
},
DataPool: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{
Size: 1,
RequireSafeReplicaSize: false,
},
},
Gateway: cephv1.GatewaySpec{
Port: 80,
Instances: 1,
},
AllowUsersInNamespaces: []string{ns.Name},
},
}
objectStoreSvc = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: objectStore.Name,
Namespace: objectStore.Namespace,
},
Spec: corev1.ServiceSpec{
Selector: map[string]string{
"app": "rook-ceph-rgw",
"rook_cluster": objectStore.Namespace,
"rook_object_store": objectStore.Name,
},
Ports: []corev1.ServicePort{
{
Name: "http",
Port: 80,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromInt(8080),
},
},
SessionAffinity: corev1.ServiceAffinityNone,
Type: corev1.ServiceTypeNodePort,
},
}
storageClass = &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
Parameters: map[string]string{
"objectStoreName": objectStore.Name,
"objectStoreNamespace": objectStore.Namespace,
},
}
// test user without any quotas set
osu1 = cephv1.CephObjectStoreUser{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-user1",
Namespace: ns.Name,
},
Spec: cephv1.ObjectStoreUserSpec{
Store: objectStore.Name,
ClusterNamespace: objectStore.Namespace,
},
}
// test user with quotas set
osu2 = cephv1.CephObjectStoreUser{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-user2",
Namespace: ns.Name,
},
Spec: cephv1.ObjectStoreUserSpec{
Store: objectStore.Name,
ClusterNamespace: objectStore.Namespace,
Quotas: &cephv1.ObjectUserQuotaSpec{
MaxBuckets: func(i int) *int { return &i }(1111),
MaxSize: resource.NewQuantity(2222, resource.DecimalSI),
MaxObjects: func(i int64) *int64 { return &i }(3333),
},
},
}
obc1 = bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-obc1",
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName + "-obc1",
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{
"bucketOwner": osu1.Name,
},
},
}
obc2 = bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-obc2",
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName + "-obc2",
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{
"bucketOwner": osu1.Name,
},
},
}
obcBogusOwner = bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-bogus-owner",
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName,
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{
"bucketOwner": defaultName + "-bogus-user",
},
},
}
)
func WaitForPodLogContainingText(k8sh *utils.K8sHelper, namespace string, selector *labels.Selector, text string, timeout time.Duration) error {
@@ -231,7 +87,98 @@ func WaitForPodLogContainingText(k8sh *utils.K8sHelper, namespace string, select
return nil
}
func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool) {
func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) {
var (
defaultName = "test-bucketowner"
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
}
storageClass = &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
Parameters: map[string]string{
"objectStoreName": objectStore.Name,
"objectStoreNamespace": objectStore.Namespace,
},
}
// test user without any quotas set
osu1 = cephv1.CephObjectStoreUser{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-user1",
Namespace: ns.Name,
},
Spec: cephv1.ObjectStoreUserSpec{
Store: objectStore.Name,
ClusterNamespace: objectStore.Namespace,
},
}
// test user with quotas set
osu2 = cephv1.CephObjectStoreUser{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-user2",
Namespace: ns.Name,
},
Spec: cephv1.ObjectStoreUserSpec{
Store: objectStore.Name,
ClusterNamespace: objectStore.Namespace,
Quotas: &cephv1.ObjectUserQuotaSpec{
MaxBuckets: func(i int) *int { return &i }(1111),
MaxSize: resource.NewQuantity(2222, resource.DecimalSI),
MaxObjects: func(i int64) *int64 { return &i }(3333),
},
},
}
obc1 = bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-obc1",
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName + "-obc1",
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{
"bucketOwner": osu1.Name,
},
},
}
obc2 = bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-obc2",
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName + "-obc2",
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{
"bucketOwner": osu1.Name,
},
},
}
obcBogusOwner = bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-bogus-owner",
Namespace: ns.Name,
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName,
StorageClassName: storageClass.Name,
AdditionalConfig: map[string]string{
"bucketOwner": defaultName + "-bogus-user",
},
},
}
)
t.Run("OBC bucketOwner", func(t *testing.T) {
if tlsEnable {
// There is lots of coverage of rgw working with tls enabled; skipping to save time.
@@ -249,31 +196,6 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
require.NoError(t, err)
})
t.Run(fmt.Sprintf("create CephObjectStore %q", objectStore.Name), func(t *testing.T) {
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{})
require.NoError(t, err)
osReady := utils.Retry(180, time.Second, "CephObjectStore is Ready", func() bool {
liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{})
if err != nil {
return false
}
if liveOs.Status == nil {
return false
}
// return liveOs.Status.Phase == "Ready"
return liveOs.Status.Phase == cephv1.ConditionReady
})
require.True(t, osReady)
})
t.Run(fmt.Sprintf("create svc %q", objectStoreSvc.Name), func(t *testing.T) {
_, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, objectStoreSvc, metav1.CreateOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("create sc %q", storageClass.Name), func(t *testing.T) {
_, err := k8sh.Clientset.StorageV1().StorageClasses().Create(ctx, storageClass, metav1.CreateOptions{})
require.NoError(t, err)
@@ -783,16 +705,6 @@ func TestObjectBucketClaimBucketOwner(t *testing.T, k8sh *utils.K8sHelper, insta
require.NoError(t, err)
})
t.Run(fmt.Sprintf("delete svc %q", objectStoreSvc.Name), func(t *testing.T) {
err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStoreSvc.Name, metav1.DeleteOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("delete CephObjectStore %q", objectStore.Name), func(t *testing.T) {
err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
require.NoError(t, err)
+123 -209
View File
@@ -32,7 +32,6 @@ import (
corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/installer"
@@ -40,176 +39,6 @@ import (
utilsns "github.com/rook/rook/tests/integration/object/util/sns"
)
var (
defaultName = "test-topickafka"
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
}
objectStore = &cephv1.CephObjectStore{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
// the CephObjectstore must be in the same ns as the CephCluster
Namespace: "object-ns",
},
Spec: cephv1.ObjectStoreSpec{
MetadataPool: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{
Size: 1,
RequireSafeReplicaSize: false,
},
},
DataPool: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{
Size: 1,
RequireSafeReplicaSize: false,
},
},
Gateway: cephv1.GatewaySpec{
Port: 80,
Instances: 1,
},
AllowUsersInNamespaces: []string{ns.Name},
},
}
objectStoreSvc = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: objectStore.Name,
Namespace: objectStore.Namespace,
},
Spec: corev1.ServiceSpec{
Selector: map[string]string{
"app": "rook-ceph-rgw",
"rook_cluster": objectStore.Namespace,
"rook_object_store": objectStore.Name,
},
Ports: []corev1.ServicePort{
{
Name: "http",
Port: 80,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromInt(8080),
},
},
SessionAffinity: corev1.ServiceAffinityNone,
Type: corev1.ServiceTypeNodePort,
},
}
secret1 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret1",
Namespace: ns.Name,
},
Data: map[string][]byte{
"user-name": []byte("kafka-user1"),
},
}
secret2 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret2",
Namespace: ns.Name,
},
Data: map[string][]byte{
"password": []byte("kafka-pass2"),
},
}
secret3 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret3",
Namespace: ns.Name,
},
Data: map[string][]byte{
"user-name": []byte("kafka-user3"),
},
}
secret4 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret4",
Namespace: ns.Name,
},
Data: map[string][]byte{
"password": []byte("kafka-pass4"),
},
}
storageClass = &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
Parameters: map[string]string{
"objectStoreName": objectStore.Name,
"objectStoreNamespace": objectStore.Namespace,
},
}
obc1 = &bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-obc1",
Namespace: ns.Name,
Labels: map[string]string{
"bucket-notification-" + bn1.Name: bn1.Name,
},
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName + "-obc1",
StorageClassName: storageClass.Name,
},
}
bt1 = &cephv1.CephBucketTopic{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-topic1",
Namespace: ns.Name,
},
Spec: cephv1.BucketTopicSpec{
ObjectStoreName: objectStore.Name,
ObjectStoreNamespace: objectStore.Namespace,
Persistent: false,
Endpoint: cephv1.TopicEndpointSpec{
Kafka: &cephv1.KafkaEndpointSpec{
URI: "kafka://kafka.example.com:9094",
AckLevel: "broker",
UseSSL: false,
UserSecretRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret1.Name,
},
Key: "user-name",
},
PasswordSecretRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret2.Name,
},
Key: "password",
},
},
},
},
}
bn1 = &cephv1.CephBucketNotification{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-notification1",
Namespace: ns.Name,
},
Spec: cephv1.BucketNotificationSpec{
Topic: bt1.Name,
Events: []cephv1.BucketNotificationEvent{
"s3:ObjectCreated:*",
},
},
}
)
func checkStatusSecrets(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBucketTopic, expectedSecrets []*corev1.Secret) {
t.Run(fmt.Sprintf("cephBucketTopic %q has .status.secrets set", bt.Name), func(t *testing.T) {
ctx := context.TODO()
@@ -301,9 +130,128 @@ func cephBucketTopicReady(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBu
return liveBt
}
func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool) {
ctx := context.TODO()
var snsClient *sns.Client
func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) {
var (
defaultName = "test-topickafka"
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
}
secret1 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret1",
Namespace: ns.Name,
},
Data: map[string][]byte{
"user-name": []byte("kafka-user1"),
},
}
secret2 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret2",
Namespace: ns.Name,
},
Data: map[string][]byte{
"password": []byte("kafka-pass2"),
},
}
secret3 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret3",
Namespace: ns.Name,
},
Data: map[string][]byte{
"user-name": []byte("kafka-user3"),
},
}
secret4 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret4",
Namespace: ns.Name,
},
Data: map[string][]byte{
"password": []byte("kafka-pass4"),
},
}
storageClass = &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
Parameters: map[string]string{
"objectStoreName": objectStore.Name,
"objectStoreNamespace": objectStore.Namespace,
},
}
bt1 = &cephv1.CephBucketTopic{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-topic1",
Namespace: ns.Name,
},
Spec: cephv1.BucketTopicSpec{
ObjectStoreName: objectStore.Name,
ObjectStoreNamespace: objectStore.Namespace,
Persistent: false,
Endpoint: cephv1.TopicEndpointSpec{
Kafka: &cephv1.KafkaEndpointSpec{
URI: "kafka://kafka.example.com:9094",
AckLevel: "broker",
UseSSL: false,
UserSecretRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret1.Name,
},
Key: "user-name",
},
PasswordSecretRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret2.Name,
},
Key: "password",
},
},
},
},
}
bn1 = &cephv1.CephBucketNotification{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-notification1",
Namespace: ns.Name,
},
Spec: cephv1.BucketNotificationSpec{
Topic: bt1.Name,
Events: []cephv1.BucketNotificationEvent{
"s3:ObjectCreated:*",
},
},
}
obc1 = &bktv1alpha1.ObjectBucketClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-obc1",
Namespace: ns.Name,
Labels: map[string]string{
"bucket-notification-" + bn1.Name: bn1.Name,
},
},
Spec: bktv1alpha1.ObjectBucketClaimSpec{
BucketName: defaultName + "-obc1",
StorageClassName: storageClass.Name,
},
}
ctx = context.TODO()
snsClient *sns.Client
)
t.Run("CephBucketTopic kafka", func(t *testing.T) {
if tlsEnable {
@@ -316,30 +264,6 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal
require.NoError(t, err)
})
t.Run(fmt.Sprintf("create CephObjectStore %q", objectStore.Name), func(t *testing.T) {
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{})
require.NoError(t, err)
osReady := utils.Retry(180, time.Second, "CephObjectStore is Ready", func() bool {
liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{})
if err != nil {
return false
}
if liveOs.Status == nil {
return false
}
return liveOs.Status.Phase == cephv1.ConditionReady
})
require.True(t, osReady)
})
t.Run(fmt.Sprintf("create svc %q", objectStoreSvc.Name), func(t *testing.T) {
_, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, objectStoreSvc, metav1.CreateOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("create secret %q", secret1.Name), func(t *testing.T) {
_, err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Create(ctx, secret1, metav1.CreateOptions{})
require.NoError(t, err)
@@ -386,7 +310,7 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal
t.Run("setup sns client", func(t *testing.T) {
var err error
snsClient, err = utilsns.NewClient(objectStore, objectStoreSvc, k8sh, installer, tlsEnable)
snsClient, err = utilsns.NewClient(objectStore, k8sh, installer, tlsEnable)
require.NoError(t, err)
})
@@ -683,16 +607,6 @@ func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *instal
assert.Len(t, secrets.Items, 0)
})
t.Run(fmt.Sprintf("delete svc %q", objectStoreSvc.Name), func(t *testing.T) {
err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStoreSvc.Name, metav1.DeleteOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("delete CephObjectStore %q", objectStore.Name), func(t *testing.T) {
err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
require.NoError(t, err)
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package userkeys
package keys
import (
"context"
@@ -28,7 +28,6 @@ import (
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/installer"
@@ -36,180 +35,9 @@ import (
utiladmin "github.com/rook/rook/tests/integration/object/util/admin"
)
var (
defaultName = "test-userkeys"
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
}
objectStore = &cephv1.CephObjectStore{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
// the CephObjectstore must be in the same ns as the CephCluster
Namespace: "object-ns",
},
Spec: cephv1.ObjectStoreSpec{
MetadataPool: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{
Size: 1,
RequireSafeReplicaSize: false,
},
},
DataPool: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{
Size: 1,
RequireSafeReplicaSize: false,
},
},
Gateway: cephv1.GatewaySpec{
Port: 80,
Instances: 1,
},
AllowUsersInNamespaces: []string{ns.Name},
},
}
objectStoreSvc = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: objectStore.Name,
Namespace: objectStore.Namespace,
},
Spec: corev1.ServiceSpec{
Selector: map[string]string{
"app": "rook-ceph-rgw",
"rook_cluster": objectStore.Namespace,
"rook_object_store": objectStore.Name,
},
Ports: []corev1.ServicePort{
{
Name: "http",
Port: 80,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromInt(8080),
},
},
SessionAffinity: corev1.ServiceAffinityNone,
Type: corev1.ServiceTypeNodePort,
},
}
secret1 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret1",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("P1ANF2BP4K2LPGOR5SBB"),
"AWS_SECRET_ACCESS_KEY": []byte("yfCHPhhOed0vJkqZsGEODyJrdKqHD09OMTWCnwjX"),
},
}
secret2 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret2",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("2I6RPUTQFMNCSYEXZ6VM"),
"AWS_SECRET_ACCESS_KEY": []byte("uY066SWPfaOVlDeYc7GJyOTfkDejyDdXrqehS6wx"),
},
}
secret3 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret3",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("J4D0P20F3EDR51OSND7Y"),
"AWS_SECRET_ACCESS_KEY": []byte("jn89OpkXNoDlIHVVQ23mE2DZgPmuDK3WVH5ExOvQ"),
},
}
secret4 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret4",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("MPZX7DG5WJWQ6VPCSLYT"),
"AWS_SECRET_ACCESS_KEY": []byte("phh7DIxnLPeSD2V6FUouhmnWrKlKRD5dBykyXozX"),
},
}
secret5 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret5",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("7TNSSANCO5KXK23IPT91"),
"AWS_SECRET_ACCESS_KEY": []byte("HksEDf0hEh3PtTvl7s9x6CyXfkWuY8eAMYAcvH5l"),
},
}
osu1 = cephv1.CephObjectStoreUser{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-user1",
Namespace: ns.Name,
},
Spec: cephv1.ObjectStoreUserSpec{
Store: objectStore.Name,
ClusterNamespace: objectStore.Namespace,
Keys: []cephv1.ObjectUserKey{
{
AccessKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret1.Name,
},
Key: "AWS_ACCESS_KEY_ID",
},
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret1.Name,
},
Key: "AWS_SECRET_ACCESS_KEY",
},
},
{
AccessKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret2.Name,
},
Key: "AWS_ACCESS_KEY_ID",
},
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret2.Name,
},
Key: "AWS_SECRET_ACCESS_KEY",
},
},
{
AccessKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret3.Name,
},
Key: "AWS_ACCESS_KEY_ID",
},
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret3.Name,
},
Key: "AWS_SECRET_ACCESS_KEY",
},
},
},
},
}
)
// generate the secret name the operator is expected to generate for the CephObjectStoreUser
func generateObjectStoreUserSecretName(osu cephv1.CephObjectStoreUser) string {
return "rook-ceph-object-user-" + osu.Namespace + "-" + osu.Name
return "rook-ceph-object-user-" + osu.Spec.Store + "-" + osu.Name
}
// find a UserKeySpec by AccessKey value
@@ -226,7 +54,7 @@ func checkStatusKeys(t *testing.T, k8sh *utils.K8sHelper, osu cephv1.CephObjectS
t.Run(fmt.Sprintf("cephObjectStoreUser %q has .status.keys set", osu.Name), func(t *testing.T) {
ctx := context.TODO()
liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(ns.Name).Get(ctx, osu.Name, metav1.GetOptions{})
liveOsu, err := k8sh.RookClientset.CephV1().CephObjectStoreUsers(osu.Namespace).Get(ctx, osu.Name, metav1.GetOptions{})
require.NoError(t, err)
require.NotNil(t, liveOsu.Status)
@@ -244,7 +72,7 @@ func checkStatusKeys(t *testing.T, k8sh *utils.K8sHelper, osu cephv1.CephObjectS
require.NoError(t, err)
// fetch the live secret for UID and ResourceVersion
liveSecret, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Get(ctx, secret.Name, metav1.GetOptions{})
liveSecret, err := k8sh.Clientset.CoreV1().Secrets(osu.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
require.NoError(t, err)
assert.Equal(t, liveSecret.Name, secretRef.Name)
@@ -284,7 +112,126 @@ func checkRgwUserKeys(t *testing.T, adminClient *admin.API, osu cephv1.CephObjec
})
}
func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool) {
func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool, objectStore *cephv1.CephObjectStore) {
var (
defaultName = "test-userkeys"
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName,
},
}
secret1 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret1",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("P1ANF2BP4K2LPGOR5SBB"),
"AWS_SECRET_ACCESS_KEY": []byte("yfCHPhhOed0vJkqZsGEODyJrdKqHD09OMTWCnwjX"),
},
}
secret2 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret2",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("2I6RPUTQFMNCSYEXZ6VM"),
"AWS_SECRET_ACCESS_KEY": []byte("uY066SWPfaOVlDeYc7GJyOTfkDejyDdXrqehS6wx"),
},
}
secret3 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret3",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("J4D0P20F3EDR51OSND7Y"),
"AWS_SECRET_ACCESS_KEY": []byte("jn89OpkXNoDlIHVVQ23mE2DZgPmuDK3WVH5ExOvQ"),
},
}
secret4 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret4",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("MPZX7DG5WJWQ6VPCSLYT"),
"AWS_SECRET_ACCESS_KEY": []byte("phh7DIxnLPeSD2V6FUouhmnWrKlKRD5dBykyXozX"),
},
}
secret5 = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-secret5",
Namespace: ns.Name,
},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("7TNSSANCO5KXK23IPT91"),
"AWS_SECRET_ACCESS_KEY": []byte("HksEDf0hEh3PtTvl7s9x6CyXfkWuY8eAMYAcvH5l"),
},
}
osu1 = cephv1.CephObjectStoreUser{
ObjectMeta: metav1.ObjectMeta{
Name: defaultName + "-user1",
Namespace: ns.Name,
},
Spec: cephv1.ObjectStoreUserSpec{
Store: objectStore.Name,
ClusterNamespace: objectStore.Namespace,
Keys: []cephv1.ObjectUserKey{
{
AccessKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret1.Name,
},
Key: "AWS_ACCESS_KEY_ID",
},
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret1.Name,
},
Key: "AWS_SECRET_ACCESS_KEY",
},
},
{
AccessKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret2.Name,
},
Key: "AWS_ACCESS_KEY_ID",
},
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret2.Name,
},
Key: "AWS_SECRET_ACCESS_KEY",
},
},
{
AccessKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret3.Name,
},
Key: "AWS_ACCESS_KEY_ID",
},
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret3.Name,
},
Key: "AWS_SECRET_ACCESS_KEY",
},
},
},
},
}
)
t.Run("ObjectStoreUser keys", func(t *testing.T) {
if tlsEnable {
// Skip testing with and without TLS to reduce test time
@@ -299,30 +246,6 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins
require.NoError(t, err)
})
t.Run(fmt.Sprintf("create CephObjectStore %q", objectStore.Name), func(t *testing.T) {
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{})
require.NoError(t, err)
osReady := utils.Retry(180, time.Second, "CephObjectStore is Ready", func() bool {
liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{})
if err != nil {
return false
}
if liveOs.Status == nil {
return false
}
return liveOs.Status.Phase == cephv1.ConditionReady
})
require.True(t, osReady)
})
t.Run(fmt.Sprintf("create svc %q", objectStoreSvc.Name), func(t *testing.T) {
_, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, objectStoreSvc, metav1.CreateOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("create secret %q", secret1.Name), func(t *testing.T) {
_, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).Create(ctx, secret1, metav1.CreateOptions{})
require.NoError(t, err)
@@ -665,16 +588,6 @@ func TestObjectStoreUserKeys(t *testing.T, k8sh *utils.K8sHelper, installer *ins
assert.Len(t, secrets.Items, 0)
})
t.Run(fmt.Sprintf("delete svc %q", objectStoreSvc.Name), func(t *testing.T) {
err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStoreSvc.Name, metav1.DeleteOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("delete CephObjectStore %q", objectStore.Name), func(t *testing.T) {
err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{})
require.NoError(t, err)
})
t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
require.NoError(t, err)
@@ -0,0 +1,129 @@
/*
Copyright 2026 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package sharedstore provides a shared CephObjectStore fixture for
// tests under tests/integration/object. A single store is created once
// and torn down after all sub-package tests complete, avoiding the
// per-package setup/teardown overhead.
package sharedstore
import (
"context"
"testing"
"time"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
// Setup creates a shared CephObjectStore and its NodePort Service in
// ObjectStoreNamespace, waits for the store to become Ready, and returns a
// teardown function that deletes both resources. The teardown function should
// be deferred by the caller.
func Setup(t *testing.T, k8sh *utils.K8sHelper) (*cephv1.CephObjectStore, func()) {
t.Helper()
ctx := context.TODO()
objectStore := &cephv1.CephObjectStore{
ObjectMeta: metav1.ObjectMeta{
Name: "test-shared",
Namespace: "object-ns",
},
Spec: cephv1.ObjectStoreSpec{
MetadataPool: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{
Size: 1,
RequireSafeReplicaSize: false,
},
},
DataPool: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{
Size: 1,
RequireSafeReplicaSize: false,
},
},
Gateway: cephv1.GatewaySpec{
Port: 80,
Instances: 1,
},
// Allow CephObjectStoreUser resources from every namespace used by
// the sub-package tests under tests/integration/object.
AllowUsersInNamespaces: []string{
"test-bucketowner",
"test-topickafka",
"test-userkeys",
"test-useropmask",
},
},
}
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
// The service name must match ObjectStoreName so that
// util/s3.GetS3Endpoint can locate it by objectStore.Name.
Name: objectStore.Name,
Namespace: objectStore.Namespace,
},
Spec: corev1.ServiceSpec{
Selector: map[string]string{
"app": "rook-ceph-rgw",
"rook_cluster": objectStore.Namespace,
"rook_object_store": objectStore.Name,
},
Ports: []corev1.ServicePort{
{
Name: "http",
Port: 80,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromInt(8080),
},
},
SessionAffinity: corev1.ServiceAffinityNone,
Type: corev1.ServiceTypeNodePort,
},
}
_, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{})
require.NoError(t, err)
osReady := utils.Retry(180, time.Second, "shared CephObjectStore is Ready", func() bool {
liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{})
if err != nil {
return false
}
if liveOs.Status == nil {
return false
}
return liveOs.Status.Phase == cephv1.ConditionReady
})
require.True(t, osReady, "shared CephObjectStore did not become Ready")
_, err = k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, svc, metav1.CreateOptions{})
require.NoError(t, err)
return objectStore, func() {
if err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}); err != nil {
t.Logf("failed to delete shared object store service: %v", err)
}
if err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{}); err != nil {
t.Logf("failed to delete shared CephObjectStore: %v", err)
}
}
}
+1 -2
View File
@@ -25,7 +25,6 @@ import (
"github.com/aws/aws-sdk-go-v2/service/sns"
smithyendpoints "github.com/aws/smithy-go/endpoints"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/tests/framework/installer"
@@ -49,7 +48,7 @@ func (r *snsResolverV2) ResolveEndpoint(ctx context.Context, params sns.Endpoint
}, nil
}
func NewClient(objectStore *cephv1.CephObjectStore, svc *corev1.Service, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) {
func NewClient(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) {
ctx := context.TODO()
accessKey, secretKey, err := utils3.GetS3Credentials(objectStore, installer)