From df3de459486d92a02404cd48deb9b580023a6f66 Mon Sep 17 00:00:00 2001 From: Artem Torubarov Date: Wed, 19 Feb 2025 12:29:13 +0100 Subject: [PATCH 1/2] object: allow overriding rgw config value from secret Implements #15422 Extends existing ObjectStore configuration options with rgwConfigFromSecret allowing to refer config value from kubernetes secret. Signed-off-by: Artem Torubarov (cherry picked from commit 3b2e6f5bf629283d51d8a309474aef9e020b9b1c) --- .../Object-Storage/ceph-object-store-crd.md | 37 +++++++++++++- Documentation/CRDs/specification.md | 17 +++++++ .../charts/rook-ceph/templates/resources.yaml | 30 ++++++++++++ deploy/examples/crds.yaml | 30 ++++++++++++ pkg/apis/ceph.rook.io/v1/types.go | 8 +++ .../ceph.rook.io/v1/zz_generated.deepcopy.go | 7 +++ pkg/operator/ceph/object/config.go | 30 +++++++++++- pkg/operator/ceph/object/config_test.go | 49 +++++++++++++++++++ 8 files changed, 205 insertions(+), 3 deletions(-) diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index cd8839836..542579d16 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -390,6 +390,10 @@ spec: debug_rgw: "10" # int # debug-rgw: "20" # equivalent config keys can have dashes or underscores rgw_s3_auth_use_ldap: "true" # bool + rgwConfigFromSecret: + rgw_keystone_barbican_password: # name of rgw option with secret value + name: "barbican-secret" # name of K8s secret + key: "password" # key of secret value in K8s secret.Data map[string]string rgwCommandFlags: rgw_dmclock_auth_res: "100.0" # float rgw_d4n_l1_datacache_persistent_path: /var/log/rook/rgwd4ncache # string @@ -397,12 +401,13 @@ spec: ``` * `rgwConfig` - These configurations are applied and modified at runtime, without RGW restart. +* `rgwConfigFromSecret` - same as `rgwConfig` but config value is referenced from k8s secret. * `rgwCommandFlags` - These configurations are applied as CLI arguments and result in RGW daemons restarting when updates are applied. Restarts are desired behavior for some RGW configs. !!! note - Once an `rgwConfig` is set, it will not be removed from Ceph's central config store when removed - from the `rgwConfig` spec. Be sure to specifically set values back to their defaults once done. + Once an `rgwConfig` or `rgwConfigFromSecret` is set, it will not be removed from Ceph's central config store when removed + from the `rgwConfig` or `rgwConfigFromSecret` spec. Be sure to specifically set values back to their defaults once done. With this in mind, `rgwCommandFlags` may be a better choice for temporary config values like debug levels. @@ -455,6 +460,34 @@ data: "bindpass.secret": aGVsbG8ud29ybGQK # hello.world ``` +### Example - usage with `rgwConfigFromSecret` + +The sample configuration below demonstrates how to securely handle secret configuration parameters when setting up RGW for [Barbican integration](https://docs.ceph.com/en/reef/radosgw/config-ref/#barbican-settings). + +```yaml + # ... + gateway: + # ... + rgwConfig: + rgw_barbican_url: "http://barbican.example.com:9311" + rgw_keystone_barbican_domain: "domain" + rgw_keystone_barbican_project: "project" + rgw_keystone_barbican_user: "user" + rgwConfigFromSecret: + rgw_keystone_barbican_password: + name: "barbican-secret" + key: "password" +--- +apiVersion: v1 +kind: Secret +metadata: + name: barbican-secret + namespace: rook-ceph +type: Opaque +data: + "password": aGVsbG8ud29ybGQK # hello.world +``` + ## Deleting a CephObjectStore During deletion of a CephObjectStore resource, Rook protects against accidental or premature diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 99e2000fd..deeee089b 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7203,6 +7203,23 @@ applied. Use with caution.

+rgwConfigFromSecret
+ + +map[string]k8s.io/api/core/v1.SecretKeySelector + + + + +(Optional) +

RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference. +Values are modified at runtime without RGW restart. +This feature is intended for advanced users. It allows breaking configurations to be easily +applied. Use with caution.

+ + + + rgwCommandFlags
map[string]string diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 53a806f02..5deee93f9 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -11953,6 +11953,36 @@ spec: applied. Use with caution. nullable: true type: object + rgwConfigFromSecret: + additionalProperties: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + description: |- + RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference. + Values are modified at runtime without RGW restart. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object securePort: description: The port the rgw service will be listening on (https) format: int32 diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 25b576af4..327b8e363 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -11944,6 +11944,36 @@ spec: applied. Use with caution. nullable: true type: object + rgwConfigFromSecret: + additionalProperties: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + description: |- + RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference. + Values are modified at runtime without RGW restart. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object securePort: description: The port the rgw service will be listening on (https) format: int32 diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index de8c70cce..586a3fe07 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1748,6 +1748,14 @@ type GatewaySpec struct { // +optional RgwConfig map[string]string `json:"rgwConfig,omitempty"` + // RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference. + // Values are modified at runtime without RGW restart. + // This feature is intended for advanced users. It allows breaking configurations to be easily + // applied. Use with caution. + // +nullable + // +optional + RgwConfigFromSecret map[string]v1.SecretKeySelector `json:"rgwConfigFromSecret,omitempty"` + // RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object // store. Values are modified at RGW startup, resulting in RGW pod restarts. // This feature is intended for advanced users. It allows breaking configurations to be easily diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 90d36b973..7065861a7 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -2743,6 +2743,13 @@ func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) { (*out)[key] = val } } + if in.RgwConfigFromSecret != nil { + in, out := &in.RgwConfigFromSecret, &out.RgwConfigFromSecret + *out = make(map[string]corev1.SecretKeySelector, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } if in.RgwCommandFlags != nil { in, out := &in.RgwCommandFlags, &out.RgwCommandFlags *out = make(map[string]string, len(*in)) diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 8c3608f2b..73044203c 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -27,6 +27,7 @@ import ( cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -215,12 +216,39 @@ func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[stri for flag, val := range c.store.Spec.Gateway.RgwConfig { if currVal, ok := configOptions[flag]; ok { // RGW might break with some user-specified config overrides; log clearly to help triage - logger.Infof("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q", + logger.Infof("overriding object store %q RGW config option %q with user-specified rgwConfig", + fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag) + logger.Debugf("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, val) } configOptions[flag] = val } + for flag, val := range c.store.Spec.Gateway.RgwConfigFromSecret { + if val.Key == "" || val.Name == "" { + logger.Warningf("invalid object store %q rgwConfigFromSecret value %q=%+v", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, val) + continue + } + secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, val.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("unable to lookup secret for object store %q rgwConfigFromSecret %q=%+v", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, val) + } + secretVal, ok := secret.Data[val.Key] + if !ok { + return nil, fmt.Errorf("unable to lookup secret for object store %q rgwConfigFromSecret %q: key %q not found in secret %q", + fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, val.Key, val.Name) + } + + if currVal, ok := configOptions[flag]; ok { + // RGW might break with some user-specified config overrides; log clearly to help triage + logger.Infof("overriding object store %q RGW config option %q with user-specified rgwConfigFromSecret", + fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag) + logger.Debugf("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfigFromSecret %q", + fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, secretVal) + } + configOptions[flag] = string(secretVal) + } + return configOptions, nil } diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go index f146ce617..21ae599c6 100644 --- a/pkg/operator/ceph/object/config_test.go +++ b/pkg/operator/ceph/object/config_test.go @@ -17,6 +17,7 @@ limitations under the License. package object import ( + "context" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -25,6 +26,8 @@ import ( cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func newConfig(t *testing.T) *clusterConfig { @@ -182,3 +185,49 @@ func Test_clusterConfig_generateMonConfigOptions(t *testing.T) { }) } } + +func TestRgwConfigFromSecret(t *testing.T) { + objContext := &Context{ + Context: &clusterd.Context{ + Clientset: test.New(t, 3), + }, + clusterInfo: cephclient.AdminTestClusterInfo("rook-ceph"), + } + + objectStore := simpleStore() + objectStore.Spec.Gateway.RgwConfigFromSecret = map[string]v1.SecretKeySelector{ + "rgw_secret_conf_name": { + LocalObjectReference: v1.LocalObjectReference{ + Name: "my-secret", + }, + Key: "secKey", + }, + } + + c := &clusterConfig{ + store: objectStore, + context: objContext.Context, + clusterInfo: objContext.clusterInfo, + } + + rgwConfig := &rgwConfig{} + + _, err := c.generateMonConfigOptions(rgwConfig) + assert.Error(t, err, "error if secret not exists") + + s := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-secret", + Namespace: "rook-ceph", + }, + Data: map[string][]byte{ + "secKey": []byte("secVal"), + }, + } + _, err = objContext.Context.Clientset.CoreV1().Secrets(objContext.clusterInfo.Namespace).Create(context.TODO(), s, metav1.CreateOptions{}) + assert.NoError(t, err, "create secret") + + got, err := c.generateMonConfigOptions(rgwConfig) + assert.NoError(t, err) + assert.EqualValues(t, "secVal", got["rgw_secret_conf_name"]) +} From 7752b7b1854a97678c1ea1b98d1bb35160375268 Mon Sep 17 00:00:00 2001 From: Artem Torubarov Date: Tue, 25 Feb 2025 14:17:49 +0100 Subject: [PATCH 2/2] object: change log level for config from secret Signed-off-by: Artem Torubarov (cherry picked from commit 81d0f69aaec8410ff9ad7c13e72070337bc9dd97) --- .../CRDs/Object-Storage/ceph-object-store-crd.md | 8 ++++---- pkg/operator/ceph/object/config.go | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index 542579d16..f04c71013 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -462,7 +462,7 @@ data: ### Example - usage with `rgwConfigFromSecret` -The sample configuration below demonstrates how to securely handle secret configuration parameters when setting up RGW for [Barbican integration](https://docs.ceph.com/en/reef/radosgw/config-ref/#barbican-settings). +The sample configuration below demonstrates how to securely handle secret configuration parameters when setting up RGW for [Barbican integration](https://docs.ceph.com/en/latest/radosgw/config-ref/#barbican-settings). ```yaml # ... @@ -481,11 +481,11 @@ The sample configuration below demonstrates how to securely handle secret config apiVersion: v1 kind: Secret metadata: - name: barbican-secret - namespace: rook-ceph + name: barbican-secret + namespace: rook-ceph type: Opaque data: - "password": aGVsbG8ud29ybGQK # hello.world + "password": aGVsbG8ud29ybGQK # hello.world ``` ## Deleting a CephObjectStore diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 73044203c..dad9879c6 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -218,7 +218,7 @@ func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[stri // RGW might break with some user-specified config overrides; log clearly to help triage logger.Infof("overriding object store %q RGW config option %q with user-specified rgwConfig", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag) - logger.Debugf("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q", + logger.Tracef("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, val) } configOptions[flag] = val @@ -226,8 +226,7 @@ func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[stri for flag, val := range c.store.Spec.Gateway.RgwConfigFromSecret { if val.Key == "" || val.Name == "" { - logger.Warningf("invalid object store %q rgwConfigFromSecret value %q=%+v", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, val) - continue + return nil, fmt.Errorf("invalid object store %q rgwConfigFromSecret value %q=%+v", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, val) } secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, val.Name, metav1.GetOptions{}) if err != nil { @@ -243,7 +242,7 @@ func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[stri // RGW might break with some user-specified config overrides; log clearly to help triage logger.Infof("overriding object store %q RGW config option %q with user-specified rgwConfigFromSecret", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag) - logger.Debugf("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfigFromSecret %q", + logger.Tracef("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfigFromSecret %q", fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, secretVal) } configOptions[flag] = string(secretVal)