Merge pull request #15772 from rook/mergify/bp/release-1.17/pr-15696

operator: specify ceph config via secret (backport #15696)

Documentation/CRDs/Cluster/ceph-cluster-crd.md

@@ -108,6 +108,7 @@ For more details on the mons and when to choose a number other than `3`, see the
 * `cleanupPolicy`: [cleanup policy settings](#cleanup-policy)
 * `security`: [security page for key management configuration](../../Storage-Configuration/Advanced/key-management-system.md)
 * `cephConfig`: [Set Ceph config options using the Ceph Mon config store](#ceph-config)
+* `cephConfigFromSecret`: [Set Ceph config options using the Ceph Mon config store via Kubernetes secret reference](#ceph-config-from-secret)
 * `csi`: [Set CSI Driver options](#csi-driver-options)
 
 ### Ceph container images
@@ -887,6 +888,29 @@ should be used instead.
 
 The operator does not unset any removed config options, it is the user's responsibility to unset or set the default value for each removed option manually using the Ceph CLI.
 
+## Ceph Config From Secret
+
+In addition to `cephConfig`, Ceph configuration values can be provided via Kubernetes Secrets using `cephConfigFromSecret`. This is useful for referencing sensitive values such as passwords or tokens that shouldn't be stored directly in the CR.
+
+The structure mirrors `cephConfig`, but instead of string values, each config entry references a Kubernetes `SecretKeySelector`, specifying:
+
+- name: the name of the secret (in the same namespace as the `CephCluster`)
+- key: the key inside the secret that holds the desired value
+
+```yaml
+spec:
+  cephConfigFromSecret:
+    mgr:
+      mgr/dashboard/GRAFANA_API_PASSWORD:
+        name: grafana-api-credential # name of the Kubernetes secret
+        key: password                # key of the secret value in Kubernetes secret.Data map[string]string
+```
+
+If a given config key is defined in both `cephConfigFromSecret` and `cephConfig`, the value from `cephConfig` takes precedence.
+
+!!! warning
+    If a value from `cephConfigFromSecret` cannot be retrieved — for example, if the referenced Secret or key is missing — Rook will return a reconciliation error. This ensures that configuration provided via `cephConfigFromSecret` is applied reliably, as it is treated as a declarative and intentional configuration by the admin.
+
 ## CSI Driver Options
 
 The CSI driver options mentioned here are applied per Ceph cluster. The following options are available:

Documentation/CRDs/specification.md

@@ -1048,6 +1048,18 @@ map[string]map[string]string
 <p>Ceph Config options</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>cephConfigFromSecret</code><br/>
+<em>
+map[string]map[string]k8s.io/api/core/v1.SecretKeySelector
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CephConfigFromSecret works exactly like CephConfig but takes config value from Secret Key reference.</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -4905,6 +4917,18 @@ map[string]map[string]string
 <p>Ceph Config options</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>cephConfigFromSecret</code><br/>
+<em>
+map[string]map[string]k8s.io/api/core/v1.SecretKeySelector
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CephConfigFromSecret works exactly like CephConfig but takes config value from Secret Key reference.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.ClusterState">ClusterState

Documentation/Storage-Configuration/Advanced/ceph-configuration.md

@@ -209,7 +209,7 @@ ceph osd pool set rbd pg_num 512
 ## Custom `ceph.conf` Settings
 
 !!! info
-    The advised method for controlling Ceph configuration is to use the [`cephConfig:` structure](../../CRDs/Cluster/ceph-cluster-crd.md#ceph-config)
+    The advised method for controlling Ceph configuration is to use the [`cephConfig`](../../CRDs/Cluster/ceph-cluster-crd.md#ceph-config) and [`cephConfigFromSecret`](../../CRDs/Cluster/ceph-cluster-crd.md#ceph-config-from-secret)
     in the `CephCluster` CRD.
     <br><br>It is highly recommended that this only be used when absolutely necessary and that the `config` be
     reset to an empty string if/when the configurations are no longer necessary. Configurations in the

deploy/charts/rook-ceph/templates/resources.yaml

@@ -1265,6 +1265,34 @@ spec:
                   description: Ceph Config options
                   nullable: true
                   type: object
+                cephConfigFromSecret:
+                  additionalProperties:
+                    additionalProperties:
+                      description: SecretKeySelector selects a key of a Secret.
+                      properties:
+                        key:
+                          description: The key of the secret to select from.  Must be a valid secret key.
+                          type: string
+                        name:
+                          default: ""
+                          description: |-
+                            Name of the referent.
+                            This field is effectively required, but due to backwards compatibility is
+                            allowed to be empty. Instances of this type with an empty value here are
+                            almost certainly wrong.
+                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                          type: string
+                        optional:
+                          description: Specify whether the Secret or its key must be defined
+                          type: boolean
+                      required:
+                        - key
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: object
+                  description: CephConfigFromSecret works exactly like CephConfig but takes config value from Secret Key reference.
+                  nullable: true
+                  type: object
                 cephVersion:
                   description: The version information that instructs Rook to orchestrate a particular version of Ceph.
                   nullable: true

deploy/examples/crds.yaml

@@ -1263,6 +1263,34 @@ spec:
                   description: Ceph Config options
                   nullable: true
                   type: object
+                cephConfigFromSecret:
+                  additionalProperties:
+                    additionalProperties:
+                      description: SecretKeySelector selects a key of a Secret.
+                      properties:
+                        key:
+                          description: The key of the secret to select from.  Must be a valid secret key.
+                          type: string
+                        name:
+                          default: ""
+                          description: |-
+                            Name of the referent.
+                            This field is effectively required, but due to backwards compatibility is
+                            allowed to be empty. Instances of this type with an empty value here are
+                            almost certainly wrong.
+                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                          type: string
+                        optional:
+                          description: Specify whether the Secret or its key must be defined
+                          type: boolean
+                      required:
+                        - key
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: object
+                  description: CephConfigFromSecret works exactly like CephConfig but takes config value from Secret Key reference.
+                  nullable: true
+                  type: object
                 cephVersion:
                   description: The version information that instructs Rook to orchestrate a particular version of Ceph.
                   nullable: true

pkg/apis/ceph.rook.io/v1/types.go

@@ -239,6 +239,11 @@ type ClusterSpec struct {
 	// +optional
 	// +nullable
 	CephConfig map[string]map[string]string `json:"cephConfig,omitempty"`
+
+	// CephConfigFromSecret works exactly like CephConfig but takes config value from Secret Key reference.
+	// +optional
+	// +nullable
+	CephConfigFromSecret map[string]map[string]v1.SecretKeySelector `json:"cephConfigFromSecret,omitempty"`
 }
 
 // CSIDriverSpec defines CSI Driver settings applied per cluster.

pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go

@@ -2046,6 +2046,23 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 			(*out)[key] = outVal
 		}
 	}
+	if in.CephConfigFromSecret != nil {
+		in, out := &in.CephConfigFromSecret, &out.CephConfigFromSecret
+		*out = make(map[string]map[string]corev1.SecretKeySelector, len(*in))
+		for key, val := range *in {
+			var outVal map[string]corev1.SecretKeySelector
+			if val == nil {
+				(*out)[key] = nil
+			} else {
+				in, out := &val, &outVal
+				*out = make(map[string]corev1.SecretKeySelector, len(*in))
+				for key, val := range *in {
+					(*out)[key] = *val.DeepCopy()
+				}
+			}
+			(*out)[key] = outVal
+		}
+	}
 	return
 }
 

pkg/operator/ceph/cluster/cluster.go

@@ -567,7 +567,17 @@ func (c *cluster) shouldSetClusterFullSettings() bool {
 
 func (c *cluster) updateConfigStoreFromCRD() error {
 	monStore := config.GetMonStore(c.context, c.ClusterInfo)
-	return monStore.SetAllMultiple(c.Spec.CephConfig)
+	cephConfigFromSecret, err := c.fetchCephConfigFromSecrets()
+	if err != nil {
+		return err
+	}
+	if err := monStore.SetAllMultiple(cephConfigFromSecret); err != nil {
+		return err
+	}
+	if err := monStore.SetAllMultiple(c.Spec.CephConfig); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (c *cluster) reportTelemetry() {
@@ -741,3 +751,43 @@ func (c *cluster) configureMsgr2() error {
 
 	return nil
 }
+
+func (c *cluster) fetchCephConfigFromSecrets() (map[string]map[string]string, error) {
+	result := make(map[string]map[string]string)
+
+	for module, keys := range c.Spec.CephConfigFromSecret {
+		result[module] = make(map[string]string)
+
+		for key, selector := range keys {
+			val, err := c.fetchSecretValue(selector)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get value for key %q in module %q from secret %q: %w",
+					key, module, selector.LocalObjectReference.Name, err)
+			}
+
+			logger.Debugf("setting Ceph config key %q in module %q from secret %q",
+				key, module, selector.LocalObjectReference.Name)
+			logger.Tracef("setting Ceph config key %q in module %q to value %q from secret %q",
+				key, module, val, selector.LocalObjectReference.Name)
+			result[module][key] = val
+		}
+	}
+
+	return result, nil
+}
+
+func (c *cluster) fetchSecretValue(selector v1.SecretKeySelector) (string, error) {
+	secret, err := c.context.Clientset.CoreV1().Secrets(c.ClusterInfo.Namespace).Get(
+		c.ClusterInfo.Context, selector.LocalObjectReference.Name, metav1.GetOptions{},
+	)
+	if err != nil {
+		return "", fmt.Errorf("failed to get secret %q: %w", selector.LocalObjectReference.Name, err)
+	}
+
+	val, ok := secret.Data[selector.Key]
+	if !ok {
+		return "", fmt.Errorf("secret %q is missing key %q", selector.LocalObjectReference.Name, selector.Key)
+	}
+
+	return string(val), nil
+}

pkg/operator/ceph/cluster/cluster_test.go

@@ -18,6 +18,7 @@ limitations under the License.
 package cluster
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"strings"
@@ -33,11 +34,14 @@ import (
 	"github.com/rook/rook/pkg/operator/ceph/cluster/telemetry"
 	"github.com/rook/rook/pkg/operator/ceph/csi"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
+	"github.com/rook/rook/pkg/operator/test"
 	testop "github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/ini.v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestPreClusterStartValidation(t *testing.T) {
@@ -431,3 +435,163 @@ func TestClusterFullSettings(t *testing.T) {
 		assert.False(t, setBackfillFullRatio)
 	})
 }
+
+func TestFetchCephConfigFromSecrets(t *testing.T) {
+	tests := []struct {
+		name                 string
+		cephConfigFromSecret map[string]map[string]v1.SecretKeySelector
+		secrets              []*v1.Secret
+		expectedOutput       map[string]map[string]string
+		expectErr            bool
+	}{
+		{
+			name: "module with valid secret and key (foo)",
+			cephConfigFromSecret: map[string]map[string]v1.SecretKeySelector{
+				"foo": {
+					"foo/bar": {
+						LocalObjectReference: v1.LocalObjectReference{Name: "secret-foo"},
+						Key:                  "val",
+					},
+				},
+			},
+			secrets: []*v1.Secret{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "secret-foo", Namespace: "rook-ceph"},
+					Data:       map[string][]byte{"val": []byte("alpha")},
+				},
+			},
+			expectedOutput: map[string]map[string]string{
+				"foo": {"foo/bar": "alpha"},
+			},
+			expectErr: false,
+		},
+		{
+			name: "module with missing key in existing secret (bar)",
+			cephConfigFromSecret: map[string]map[string]v1.SecretKeySelector{
+				"bar": {
+					"bar/baz": {
+						LocalObjectReference: v1.LocalObjectReference{Name: "secret-bar"},
+						Key:                  "val",
+					},
+				},
+			},
+			secrets: []*v1.Secret{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "secret-bar", Namespace: "rook-ceph"},
+					Data:       map[string][]byte{"not-val": []byte("noop")},
+				},
+			},
+			expectErr: true,
+		},
+		{
+			name: "module with missing secret (bar)",
+			cephConfigFromSecret: map[string]map[string]v1.SecretKeySelector{
+				"bar": {
+					"bar/baz": {
+						LocalObjectReference: v1.LocalObjectReference{Name: "missing-secret"},
+						Key:                  "val",
+					},
+				},
+			},
+			secrets:   []*v1.Secret{},
+			expectErr: true,
+		},
+		{
+			name: "multiple modules with valid keys",
+			cephConfigFromSecret: map[string]map[string]v1.SecretKeySelector{
+				"foo": {
+					"foo/key1": {
+						LocalObjectReference: v1.LocalObjectReference{Name: "secret-1"},
+						Key:                  "k1",
+					},
+				},
+				"bar": {
+					"bar/key2": {
+						LocalObjectReference: v1.LocalObjectReference{Name: "secret-2"},
+						Key:                  "k2",
+					},
+				},
+			},
+			secrets: []*v1.Secret{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "secret-1", Namespace: "rook-ceph"},
+					Data:       map[string][]byte{"k1": []byte("v1")},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "secret-2", Namespace: "rook-ceph"},
+					Data:       map[string][]byte{"k2": []byte("v2")},
+				},
+			},
+			expectedOutput: map[string]map[string]string{
+				"foo": {"foo/key1": "v1"},
+				"bar": {"bar/key2": "v2"},
+			},
+			expectErr: false,
+		},
+		{
+			name: "multiple modules with one valid key and one missing key",
+			cephConfigFromSecret: map[string]map[string]v1.SecretKeySelector{
+				"foo": {
+					"foo/key": {
+						LocalObjectReference: v1.LocalObjectReference{Name: "secret-foo"},
+						Key:                  "valid-key",
+					},
+				},
+				"bar": {
+					"bar/key": {
+						LocalObjectReference: v1.LocalObjectReference{Name: "secret-bar"},
+						Key:                  "missing-key",
+					},
+				},
+			},
+			secrets: []*v1.Secret{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "secret-foo", Namespace: "rook-ceph"},
+					Data:       map[string][]byte{"valid-key": []byte("ok")},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "secret-bar", Namespace: "rook-ceph"},
+					Data:       map[string][]byte{"not-the-key": []byte("noop")},
+				},
+			},
+			expectErr: true,
+		},
+		{
+			name:                 "cephConfigFromSecret is nil",
+			cephConfigFromSecret: nil,
+			secrets:              []*v1.Secret{},
+			expectedOutput:       map[string]map[string]string{},
+			expectErr:            false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			clientset := test.New(t, 1)
+			clusterInfo := cephclient.AdminTestClusterInfo("rook-ceph")
+
+			for _, s := range tc.secrets {
+				_, err := clientset.CoreV1().Secrets(s.Namespace).Create(context.TODO(), s, metav1.CreateOptions{})
+				assert.NoError(t, err)
+			}
+
+			c := &cluster{
+				context: &clusterd.Context{
+					Clientset: clientset,
+				},
+				ClusterInfo: clusterInfo,
+				Spec: &cephv1.ClusterSpec{
+					CephConfigFromSecret: tc.cephConfigFromSecret,
+				},
+			}
+
+			result, err := c.fetchCephConfigFromSecrets()
+			if tc.expectErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tc.expectedOutput, result)
+			}
+		})
+	}
+}