Merge pull request #15489 from rook/mergify/bp/release-1.16/pr-15426

object: allow overriding rgw config value from secret (backport #15426)

Documentation/CRDs/Object-Storage/ceph-object-store-crd.md

@@ -390,6 +390,10 @@ spec:
       debug_rgw: "10" # int
       # debug-rgw: "20" # equivalent config keys can have dashes or underscores
       rgw_s3_auth_use_ldap: "true" # bool
+    rgwConfigFromSecret:
+      rgw_keystone_barbican_password: # name of rgw option with secret value
+        name: "barbican-secret" # name of K8s secret 
+        key: "password" # key of secret value in K8s secret.Data map[string]string
     rgwCommandFlags:
       rgw_dmclock_auth_res: "100.0" # float
       rgw_d4n_l1_datacache_persistent_path: /var/log/rook/rgwd4ncache # string
@@ -397,12 +401,13 @@ spec:
 ```
 
 * `rgwConfig` - These configurations are applied and modified at runtime, without RGW restart.
+* `rgwConfigFromSecret` - same as `rgwConfig` but config value is referenced from k8s secret.
 * `rgwCommandFlags` - These configurations are applied as CLI arguments and result in RGW daemons
     restarting when updates are applied. Restarts are desired behavior for some RGW configs.
 
 !!! note
-    Once an `rgwConfig` is set, it will not be removed from Ceph's central config store when removed
-    from the `rgwConfig` spec. Be sure to specifically set values back to their defaults once done.
+    Once an `rgwConfig` or `rgwConfigFromSecret` is set, it will not be removed from Ceph's central config store when removed
+    from the `rgwConfig` or `rgwConfigFromSecret` spec. Be sure to specifically set values back to their defaults once done.
     With this in mind, `rgwCommandFlags` may be a better choice for temporary config values like
     debug levels.
 
@@ -455,6 +460,34 @@ data:
     "bindpass.secret": aGVsbG8ud29ybGQK # hello.world
 ```
 
+### Example - usage with `rgwConfigFromSecret`
+
+The sample configuration below demonstrates how to securely handle secret configuration parameters when setting up RGW for [Barbican integration](https://docs.ceph.com/en/latest/radosgw/config-ref/#barbican-settings).
+
+```yaml
+  # ...
+  gateway:
+    # ...
+    rgwConfig:
+      rgw_barbican_url: "http://barbican.example.com:9311"
+      rgw_keystone_barbican_domain: "domain"
+      rgw_keystone_barbican_project: "project" 
+      rgw_keystone_barbican_user: "user"
+    rgwConfigFromSecret:
+      rgw_keystone_barbican_password:
+        name: "barbican-secret"
+        key: "password"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: barbican-secret
+  namespace: rook-ceph
+type: Opaque
+data:
+  "password": aGVsbG8ud29ybGQK # hello.world
+```
+
 ## Deleting a CephObjectStore
 
 During deletion of a CephObjectStore resource, Rook protects against accidental or premature

Documentation/CRDs/specification.md

@@ -7203,6 +7203,23 @@ applied. Use with caution.</p>
 </tr>
 <tr>
 <td>
+<code>rgwConfigFromSecret</code><br/>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core">
+map[string]k8s.io/api/core/v1.SecretKeySelector
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference.
+Values are modified at runtime without RGW restart.
+This feature is intended for advanced users. It allows breaking configurations to be easily
+applied. Use with caution.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>rgwCommandFlags</code><br/>
 <em>
 map[string]string

deploy/charts/rook-ceph/templates/resources.yaml

@@ -11953,6 +11953,36 @@ spec:
                         applied. Use with caution.
                       nullable: true
                       type: object
+                    rgwConfigFromSecret:
+                      additionalProperties:
+                        description: SecretKeySelector selects a key of a Secret.
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must be a valid secret key.
+                            type: string
+                          name:
+                            default: ""
+                            description: |-
+                              Name of the referent.
+                              This field is effectively required, but due to backwards compatibility is
+                              allowed to be empty. Instances of this type with an empty value here are
+                              almost certainly wrong.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must be defined
+                            type: boolean
+                        required:
+                          - key
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      description: |-
+                        RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference.
+                        Values are modified at runtime without RGW restart.
+                        This feature is intended for advanced users. It allows breaking configurations to be easily
+                        applied. Use with caution.
+                      nullable: true
+                      type: object
                     securePort:
                       description: The port the rgw service will be listening on (https)
                       format: int32

deploy/examples/crds.yaml

@@ -11944,6 +11944,36 @@ spec:
                         applied. Use with caution.
                       nullable: true
                       type: object
+                    rgwConfigFromSecret:
+                      additionalProperties:
+                        description: SecretKeySelector selects a key of a Secret.
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must be a valid secret key.
+                            type: string
+                          name:
+                            default: ""
+                            description: |-
+                              Name of the referent.
+                              This field is effectively required, but due to backwards compatibility is
+                              allowed to be empty. Instances of this type with an empty value here are
+                              almost certainly wrong.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must be defined
+                            type: boolean
+                        required:
+                          - key
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      description: |-
+                        RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference.
+                        Values are modified at runtime without RGW restart.
+                        This feature is intended for advanced users. It allows breaking configurations to be easily
+                        applied. Use with caution.
+                      nullable: true
+                      type: object
                     securePort:
                       description: The port the rgw service will be listening on (https)
                       format: int32

pkg/apis/ceph.rook.io/v1/types.go

@@ -1748,6 +1748,14 @@ type GatewaySpec struct {
 	// +optional
 	RgwConfig map[string]string `json:"rgwConfig,omitempty"`
 
+	// RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference.
+	// Values are modified at runtime without RGW restart.
+	// This feature is intended for advanced users. It allows breaking configurations to be easily
+	// applied. Use with caution.
+	// +nullable
+	// +optional
+	RgwConfigFromSecret map[string]v1.SecretKeySelector `json:"rgwConfigFromSecret,omitempty"`
+
 	// RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object
 	// store. Values are modified at RGW startup, resulting in RGW pod restarts.
 	// This feature is intended for advanced users. It allows breaking configurations to be easily

pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go

@@ -2743,6 +2743,13 @@ func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.RgwConfigFromSecret != nil {
+		in, out := &in.RgwConfigFromSecret, &out.RgwConfigFromSecret
+		*out = make(map[string]corev1.SecretKeySelector, len(*in))
+		for key, val := range *in {
+			(*out)[key] = *val.DeepCopy()
+		}
+	}
 	if in.RgwCommandFlags != nil {
 		in, out := &in.RgwCommandFlags, &out.RgwCommandFlags
 		*out = make(map[string]string, len(*in))

pkg/operator/ceph/object/config.go

@@ -27,6 +27,7 @@ import (
 	cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
@@ -215,12 +216,38 @@ func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[stri
 	for flag, val := range c.store.Spec.Gateway.RgwConfig {
 		if currVal, ok := configOptions[flag]; ok {
 			// RGW might break with some user-specified config overrides; log clearly to help triage
-			logger.Infof("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q",
+			logger.Infof("overriding object store %q RGW config option %q with user-specified rgwConfig",
+				fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag)
+			logger.Tracef("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q",
 				fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, val)
 		}
 		configOptions[flag] = val
 	}
 
+	for flag, val := range c.store.Spec.Gateway.RgwConfigFromSecret {
+		if val.Key == "" || val.Name == "" {
+			return nil, fmt.Errorf("invalid object store %q rgwConfigFromSecret value %q=%+v", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, val)
+		}
+		secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, val.Name, metav1.GetOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("unable to lookup secret for object store %q rgwConfigFromSecret %q=%+v", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, val)
+		}
+		secretVal, ok := secret.Data[val.Key]
+		if !ok {
+			return nil, fmt.Errorf("unable to lookup secret for object store %q rgwConfigFromSecret %q: key %q not found in secret %q",
+				fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, val.Key, val.Name)
+		}
+
+		if currVal, ok := configOptions[flag]; ok {
+			// RGW might break with some user-specified config overrides; log clearly to help triage
+			logger.Infof("overriding object store %q RGW config option %q with user-specified rgwConfigFromSecret",
+				fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag)
+			logger.Tracef("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfigFromSecret %q",
+				fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, secretVal)
+		}
+		configOptions[flag] = string(secretVal)
+	}
+
 	return configOptions, nil
 }
 

pkg/operator/ceph/object/config_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package object
 
 import (
+	"context"
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -25,6 +26,8 @@ import (
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/test"
 	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func newConfig(t *testing.T) *clusterConfig {
@@ -182,3 +185,49 @@ func Test_clusterConfig_generateMonConfigOptions(t *testing.T) {
 		})
 	}
 }
+
+func TestRgwConfigFromSecret(t *testing.T) {
+	objContext := &Context{
+		Context: &clusterd.Context{
+			Clientset: test.New(t, 3),
+		},
+		clusterInfo: cephclient.AdminTestClusterInfo("rook-ceph"),
+	}
+
+	objectStore := simpleStore()
+	objectStore.Spec.Gateway.RgwConfigFromSecret = map[string]v1.SecretKeySelector{
+		"rgw_secret_conf_name": {
+			LocalObjectReference: v1.LocalObjectReference{
+				Name: "my-secret",
+			},
+			Key: "secKey",
+		},
+	}
+
+	c := &clusterConfig{
+		store:       objectStore,
+		context:     objContext.Context,
+		clusterInfo: objContext.clusterInfo,
+	}
+
+	rgwConfig := &rgwConfig{}
+
+	_, err := c.generateMonConfigOptions(rgwConfig)
+	assert.Error(t, err, "error if secret not exists")
+
+	s := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-secret",
+			Namespace: "rook-ceph",
+		},
+		Data: map[string][]byte{
+			"secKey": []byte("secVal"),
+		},
+	}
+	_, err = objContext.Context.Clientset.CoreV1().Secrets(objContext.clusterInfo.Namespace).Create(context.TODO(), s, metav1.CreateOptions{})
+	assert.NoError(t, err, "create secret")
+
+	got, err := c.generateMonConfigOptions(rgwConfig)
+	assert.NoError(t, err)
+	assert.EqualValues(t, "secVal", got["rgw_secret_conf_name"])
+}