diff --git a/deploy/examples/import-external-cluster.sh b/deploy/examples/import-external-cluster.sh index 4f72a2d35..e2f37314d 100644 --- a/deploy/examples/import-external-cluster.sh +++ b/deploy/examples/import-external-cluster.sh @@ -243,6 +243,7 @@ function importCsiRBDProvisionerSecret() { "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" \ -p "{\"stringData\":{\"userID\":\"$userID\",\"userKey\":\"$CSI_RBD_PROVISIONER_SECRET\"}}" fi + $KUBECTL -n "$NAMESPACE" annotate secret "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" --overwrite csi.rook.io/RBDProvisionerSecret="true" } function importCsiCephFSNodeSecret() { @@ -285,6 +286,7 @@ function importCsiCephFSProvisionerSecret() { "rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME" \ -p "{\"stringData\":{\"userID\":\"$userID\",\"userKey\":\"$CSI_CEPHFS_PROVISIONER_SECRET\"}}" fi + $KUBECTL -n "$NAMESPACE" annotate secret "rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME" --overwrite csi.rook.io/CephFSProvisionerSecret="true" } function importRGWAdminOpsUser() { diff --git a/pkg/operator/ceph/csi/config.go b/pkg/operator/ceph/csi/config.go index 8b5857566..5e16f1e8f 100644 --- a/pkg/operator/ceph/csi/config.go +++ b/pkg/operator/ceph/csi/config.go @@ -54,6 +54,11 @@ const ( func CreateUpdateClientProfileRadosNamespace(ctx context.Context, c client.Client, clusterInfo *cephclient.ClusterInfo, cephBlockPoolRadosNamespaceName, clusterID string) error { logger.Info("creating ceph-csi clientProfile CR for rados namespace") + rbdSecretName, err := getSecretNameByAnnotation(c, ctx, clusterInfo.Namespace, "csi.rook.io/RBDProvisionerSecret", CsiRBDProvisionerSecret) + if err != nil { + return err + } + csiOpClientProfile := &csiopv1.ClientProfile{} csiOpClientProfile.Name = clusterID csiOpClientProfile.Namespace = os.Getenv(k8sutil.PodNamespaceEnvVar) @@ -65,7 +70,7 @@ func CreateUpdateClientProfileRadosNamespace(ctx context.Context, c client.Clien RadosNamespace: cephBlockPoolRadosNamespaceName, CephCsiSecrets: &csiopv1.CephCsiSecretsSpec{ ControllerPublishSecret: v1.SecretReference{ - Name: CsiRBDProvisionerSecret, + Name: rbdSecretName, Namespace: clusterInfo.Namespace, }, }, @@ -78,12 +83,20 @@ func CreateUpdateClientProfileRadosNamespace(ctx context.Context, c client.Clien func CreateUpdateClientProfileSubVolumeGroup(ctx context.Context, c client.Client, clusterInfo *cephclient.ClusterInfo, cephFilesystemSubVolumeGroupName, clusterID string, csiMetadataRadosNamespace string) error { logger.Info("Creating ceph-csi clientProfile CR for subvolume group") - csiOpClientProfile := generateProfileSubVolumeGroupSpec(clusterInfo, cephFilesystemSubVolumeGroupName, clusterID, csiMetadataRadosNamespace) + csiOpClientProfile, err := generateProfileSubVolumeGroupSpec(c, clusterInfo, cephFilesystemSubVolumeGroupName, clusterID, csiMetadataRadosNamespace) + if err != nil { + return err + } return createUpdateClientProfile(c, clusterInfo, csiOpClientProfile) } -func generateProfileSubVolumeGroupSpec(clusterInfo *cephclient.ClusterInfo, cephFilesystemSubVolumeGroupName, clusterID string, csiMetadataRadosNamespace string) *csiopv1.ClientProfile { +func generateProfileSubVolumeGroupSpec(c client.Client, clusterInfo *cephclient.ClusterInfo, cephFilesystemSubVolumeGroupName, clusterID string, csiMetadataRadosNamespace string) (*csiopv1.ClientProfile, error) { + cephFSSecretName, err := getSecretNameByAnnotation(c, clusterInfo.Context, clusterInfo.Namespace, "csi.rook.io/CephFSProvisionerSecret", CsiCephFSProvisionerSecret) + if err != nil { + return nil, err + } + csiOpClientProfile := &csiopv1.ClientProfile{} csiOpClientProfile.Name = clusterID csiOpClientProfile.Namespace = os.Getenv(k8sutil.PodNamespaceEnvVar) @@ -96,7 +109,7 @@ func generateProfileSubVolumeGroupSpec(clusterInfo *cephclient.ClusterInfo, ceph RadosNamespace: &csiMetadataRadosNamespace, CephCsiSecrets: &csiopv1.CephCsiSecretsSpec{ ControllerPublishSecret: v1.SecretReference{ - Name: CsiCephFSProvisionerSecret, + Name: cephFSSecretName, Namespace: clusterInfo.Namespace, }, }, @@ -105,7 +118,21 @@ func generateProfileSubVolumeGroupSpec(clusterInfo *cephclient.ClusterInfo, ceph applyCephFSMountOptions(clusterInfo, csiOpClientProfile.Spec.CephFs) - return csiOpClientProfile + return csiOpClientProfile, nil +} + +func getSecretNameByAnnotation(c client.Client, ctx context.Context, namespace, annotationKey, defaultName string) (string, error) { + secretList := &v1.SecretList{} + err := c.List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return "", errors.Wrapf(err, "failed to list secrets in namespace %q", namespace) + } + for _, secret := range secretList.Items { + if val, ok := secret.Annotations[annotationKey]; ok && val == "true" { + return secret.Name, nil + } + } + return defaultName, nil } // CreateDefaultClientProfile creates a default client profile for csi-operator to connect driver diff --git a/pkg/operator/ceph/csi/config_test.go b/pkg/operator/ceph/csi/config_test.go index 84f1efdaf..899730201 100644 --- a/pkg/operator/ceph/csi/config_test.go +++ b/pkg/operator/ceph/csi/config_test.go @@ -27,6 +27,8 @@ import ( clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -57,10 +59,11 @@ func TestCreateUpdateClientProfile(t *testing.T) { cephSubVolGrpNamespacedName := types.NamespacedName{Namespace: ns, Name: "cephSubVolumeGroupNames"} cephSubVolGrpRadosNamespaceNamespacedName := types.NamespacedName{Namespace: ns, Name: "radosNamespaceName"} csiOpClientProfile := &csiopv1.ClientProfile{} + secretsList := &v1.SecretList{} // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, csiOpClientProfile) + s.AddKnownTypes(cephv1.SchemeGroupVersion, csiOpClientProfile, secretsList) object := []runtime.Object{ csiOpClientProfile, } @@ -84,6 +87,94 @@ func TestCreateUpdateClientProfile(t *testing.T) { assert.Equal(t, *csiOpClientProfile.Spec.CephFs.RadosNamespace, cephSubVolGrpRadosNamespaceNamespacedName.Name) } +func TestGetSecretNameByAnnotation(t *testing.T) { + ns := "test-ns" + annotationKey := "csi.rook.io/RBDProvisionerSecret" + defaultName := "rook-csi-rbd-provisioner" + + t.Run("return matching secret name when annotation value is true", func(t *testing.T) { + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-rbd-secret", + Namespace: ns, + Annotations: map[string]string{ + annotationKey: "true", + }, + }, + } + cl := fake.NewClientBuilder().WithObjects(secret).Build() + + name, err := getSecretNameByAnnotation(cl, context.TODO(), ns, annotationKey, defaultName) + assert.NoError(t, err) + assert.Equal(t, "custom-rbd-secret", name) + }) + + t.Run("return default name when no secret has the annotation", func(t *testing.T) { + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unrelated-secret", + Namespace: ns, + }, + } + cl := fake.NewClientBuilder().WithObjects(secret).Build() + + name, err := getSecretNameByAnnotation(cl, context.TODO(), ns, annotationKey, defaultName) + assert.NoError(t, err) + assert.Equal(t, defaultName, name) + }) + + t.Run("return default name when annotation value is not true", func(t *testing.T) { + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-secret", + Namespace: ns, + Annotations: map[string]string{ + annotationKey: "false", + }, + }, + } + cl := fake.NewClientBuilder().WithObjects(secret).Build() + + name, err := getSecretNameByAnnotation(cl, context.TODO(), ns, annotationKey, defaultName) + assert.NoError(t, err) + assert.Equal(t, defaultName, name) + }) + + t.Run("return default name when no secrets exist", func(t *testing.T) { + cl := fake.NewClientBuilder().Build() + + name, err := getSecretNameByAnnotation(cl, context.TODO(), ns, annotationKey, defaultName) + assert.NoError(t, err) + assert.Equal(t, defaultName, name) + }) + + t.Run("return first matching secret when multiple match", func(t *testing.T) { + secret1 := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first-secret", + Namespace: ns, + Annotations: map[string]string{ + annotationKey: "true", + }, + }, + } + secret2 := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-secret", + Namespace: ns, + Annotations: map[string]string{ + annotationKey: "true", + }, + }, + } + cl := fake.NewClientBuilder().WithObjects(secret1, secret2).Build() + + name, err := getSecretNameByAnnotation(cl, context.TODO(), ns, annotationKey, defaultName) + assert.NoError(t, err) + assert.Contains(t, []string{"first-secret", "second-secret"}, name) + }) +} + func TestParseMountOptions(t *testing.T) { t.Run("single mount option", func(t *testing.T) { options := "ms_mode=crc" diff --git a/pkg/operator/ceph/file/subvolumegroup/controller_test.go b/pkg/operator/ceph/file/subvolumegroup/controller_test.go index a1a6af4f8..ccfe9db9c 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller_test.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller_test.go @@ -105,7 +105,7 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephClient{}, &cephv1.CephClusterList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephClient{}, &cephv1.CephClusterList{}, &v1.SecretList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() @@ -142,7 +142,7 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { }, }, } - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &v1.SecretList{}) cephFilesystem := &cephv1.CephFilesystem{ ObjectMeta: metav1.ObjectMeta{ @@ -223,7 +223,7 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}, &v1.SecretList{}) // Create a ReconcileCephFilesystemSubVolumeGroup object with the scheme and fake client. r = &ReconcileCephFilesystemSubVolumeGroup{ client: cl, @@ -259,7 +259,7 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { cephCluster.Spec.External.Enable = true csiOpClientProfile := &csiopv1.ClientProfile{} - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}, &v1.SecretList{}) objects := []runtime.Object{ cephFilesystemSubVolumeGroup, cephCluster, @@ -311,7 +311,7 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) { cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build() c.Client = cl - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &v1.ConfigMap{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &v1.ConfigMap{}, &v1.SecretList{}) // Create a ReconcileCephFilesystemSubVolumeGroup object with the scheme and fake client. r = &ReconcileCephFilesystemSubVolumeGroup{ client: cl, diff --git a/pkg/operator/ceph/pool/radosnamespace/controller_test.go b/pkg/operator/ceph/pool/radosnamespace/controller_test.go index ee36cc0f0..a4f735c76 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller_test.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller_test.go @@ -106,7 +106,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephClient{}, &cephv1.CephClusterList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephClient{}, &cephv1.CephClusterList{}, &v1.SecretList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() @@ -152,7 +152,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { }, }, } - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &v1.SecretList{}) t.Run("error - no ceph cluster", func(t *testing.T) { // Create pool for updating status @@ -241,7 +241,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}, &v1.SecretList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ client: cl, @@ -277,7 +277,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { cephCluster.Spec.External.Enable = true csiOpClientProfile := &csiopv1.ClientProfile{} - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &csiopv1.ClientProfile{}, &v1.SecretList{}) objects := []runtime.Object{ cephBlockPoolRadosNamespace, cephCluster, @@ -350,7 +350,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &v1.SecretList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ client: cl, @@ -400,7 +400,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &v1.SecretList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ client: cl, @@ -454,7 +454,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &v1.SecretList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ client: cl, @@ -508,7 +508,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &v1.SecretList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ client: cl, @@ -564,7 +564,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &v1.SecretList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ client: cl, @@ -624,7 +624,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { } c.Executor = executor - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}, &v1.SecretList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ client: cl,