From aaa16488de9bf79c108ab31758bb98f752811b6a Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Thu, 12 Dec 2024 15:09:39 -0700 Subject: [PATCH] object: allow overriding rgw configs and flags Implement #15119 Allow users to override RGW configurations by specifying Ceph config options in the CephObjectStore. For configurations that require RGW to be restarted when the config is applied, allow configs to be specified as CLI arguments to the RGW as well. This is an advanced option and is documented as such. Users should be careful to understand the values they are setting, as there is no validation to prevent the object store from breaking when these configs are used. Signed-off-by: Blaine Gardner --- .../Object-Storage/ceph-object-store-crd.md | 58 +++++++++++++ Documentation/CRDs/specification.md | 30 +++++++ .../charts/rook-ceph/templates/resources.yaml | 20 +++++ deploy/examples/crds.yaml | 20 +++++ pkg/apis/ceph.rook.io/v1/types.go | 16 ++++ .../ceph.rook.io/v1/zz_generated.deepcopy.go | 14 +++ pkg/operator/ceph/object/config.go | 28 ++++-- pkg/operator/ceph/object/config_test.go | 87 +++++++++++++++++++ pkg/operator/ceph/object/rgw_test.go | 28 ------ pkg/operator/ceph/object/spec.go | 6 ++ pkg/operator/ceph/object/spec_test.go | 81 +++++++++++++++++ 11 files changed, 354 insertions(+), 34 deletions(-) diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index f8569de11..ec1f60a4f 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -349,6 +349,64 @@ vault write -f transit/keys/ exportable=true # transit engine * `tokenSecretName` can be (and often will be) the same for both kms and s3 configurations. +## Advanced configuration + +!!! warning + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + +CephObjectStore allows arbitrary Ceph configurations to be applied to RGW daemons that serve the +object store. [RGW config reference](https://docs.ceph.com/en/latest/radosgw/config-ref/). + +Configurations are applied to all RGWs that serve the CephObjectStore. Values must be strings. +Below is an example showing how different RGW configs and values might be applied. The example is +intended only to show a selection of value data types. + +```yaml +# THIS SAMPLE IS NOT A RECOMMENDATION +# ... +spec: + gateway: + # ... + rgwConfig: + debug_rgw: "10" # int + # debug-rgw: "20" # equivalent config keys can have dashes or underscores + rgw_s3_auth_use_ldap: "true" # bool + rgwCommandFlags: + rgw_dmclock_auth_res: "100.0" # float + rgw_d4n_l1_datacache_persistent_path: /var/log/rook/rgwd4ncache # string + rgw_d4n_address: "127.0.0.1:6379" # IP string +``` + +* `rgwConfig` - These configurations are applied and modified at runtime, without RGW restart. +* `rgwCommandFlags` - These configurations are applied as CLI arguments and result in RGW daemons + restarting when updates are applied. Restarts are desired behavior for some RGW configs. + +!!! note + Once an `rgwConfig` is set, it will not be removed from Ceph's central config store when removed + from the `rgwConfig` spec. Be sure to specifically set values back to their defaults once done. + With this in mind, `rgwCommandFlags` may be a better choice for temporary config values like + debug levels. + +### Example - debugging + +Users are often asked to provide RGW logs at a high log level when troubleshooting complex issues. +Apply log levels to RGWs easily using `rgwCommandFlags`. + +This spec will restart the RGW(s) with the highest level debugging enabled. + +```yaml +# ... +spec: + gateway: + # ... + rgwCommandFlags: + debug_ms: "20" + debug_rgw: "20" +``` + +Once RGW debug logging is no longer needed, the values can simply be removed from the spec. + ## Deleting a CephObjectStore During deletion of a CephObjectStore resource, Rook protects against accidental or premature diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 9530f3501..c086097bf 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7172,6 +7172,36 @@ Example: for an additional mount at subPath ldap, mounted from a se bindpass.secret, the file would reside at /var/rgw/ldap/bindpass.secret.

+ + +rgwConfig
+ +map[string]string + + + +(Optional) +

RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store. +Values are modified at runtime without RGW restart. +This feature is intended for advanced users. It allows breaking configurations to be easily +applied. Use with caution.

+ + + + +rgwCommandFlags
+ +map[string]string + + + +(Optional) +

RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object +store. Values are modified at RGW startup, resulting in RGW pod restarts. +This feature is intended for advanced users. It allows breaking configurations to be easily +applied. Use with caution.

+ +

HTTPEndpointSpec diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 37d9ed907..156fee79f 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -11869,6 +11869,26 @@ spec: type: object type: object x-kubernetes-preserve-unknown-fields: true + rgwCommandFlags: + additionalProperties: + type: string + description: |- + RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object + store. Values are modified at RGW startup, resulting in RGW pod restarts. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object + rgwConfig: + additionalProperties: + type: string + description: |- + RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store. + Values are modified at runtime without RGW restart. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object securePort: description: The port the rgw service will be listening on (https) format: int32 diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 67a17ec1a..3f557e0c8 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -11860,6 +11860,26 @@ spec: type: object type: object x-kubernetes-preserve-unknown-fields: true + rgwCommandFlags: + additionalProperties: + type: string + description: |- + RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object + store. Values are modified at RGW startup, resulting in RGW pod restarts. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object + rgwConfig: + additionalProperties: + type: string + description: |- + RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store. + Values are modified at runtime without RGW restart. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object securePort: description: The port the rgw service will be listening on (https) format: int32 diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 2ef5e2858..2f5804082 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1734,6 +1734,22 @@ type GatewaySpec struct { // Example: for an additional mount at subPath `ldap`, mounted from a secret that has key // `bindpass.secret`, the file would reside at `/var/rgw/ldap/bindpass.secret`. AdditionalVolumeMounts AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"` + + // RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store. + // Values are modified at runtime without RGW restart. + // This feature is intended for advanced users. It allows breaking configurations to be easily + // applied. Use with caution. + // +nullable + // +optional + RgwConfig map[string]string `json:"rgwConfig,omitempty"` + + // RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object + // store. Values are modified at RGW startup, resulting in RGW pod restarts. + // This feature is intended for advanced users. It allows breaking configurations to be easily + // applied. Use with caution. + // +nullable + // +optional + RgwCommandFlags map[string]string `json:"rgwCommandFlags,omitempty"` } // EndpointAddress is a tuple that describes a single IP address or host name. This is a subset of diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index b63b72756..38f4fbd25 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -2731,6 +2731,20 @@ func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.RgwConfig != nil { + in, out := &in.RgwConfig, &out.RgwConfig + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.RgwCommandFlags != nil { + in, out := &in.RgwCommandFlags, &out.RgwCommandFlags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 983589d0d..682aa5d2d 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -156,6 +156,20 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { monStore := cephconfig.GetMonStore(c.context, c.clusterInfo) who := generateCephXUser(rgwConfig.ResourceName) + + configOptions, err := c.generateMonConfigOptions(rgwConfig) + if err != nil { + return err + } + + if err := monStore.SetAll(who, configOptions); err != nil { + return errors.Wrapf(err, "failed to set all RGW configs on %q", who) + } + + return nil +} + +func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[string]string, error) { configOptions := make(map[string]string) configOptions["rgw_run_sync_thread"] = "true" @@ -171,7 +185,7 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { configOptions, err := configureKeystoneAuthentication(rgwConfig, configOptions) if err != nil { - return err + return configOptions, err } if s3 := rgwConfig.Protocols.S3; s3 != nil { @@ -192,14 +206,16 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { } } - for flag, val := range configOptions { - err := monStore.Set(who, flag, val) - if err != nil { - return errors.Wrapf(err, "failed to set %q to %q on %q", flag, val, who) + for flag, val := range c.store.Spec.Gateway.RgwConfig { + if currVal, ok := configOptions[flag]; ok { + // RGW might break with some user-specified config overrides; log clearly to help triage + logger.Infof("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q", + fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, val) } + configOptions[flag] = val } - return nil + return configOptions, nil } func configureKeystoneAuthentication(rgwConfig *rgwConfig, configOptions map[string]string) (map[string]string, error) { diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go index 88d3ad9d8..d0ba361c2 100644 --- a/pkg/operator/ceph/object/config_test.go +++ b/pkg/operator/ceph/object/config_test.go @@ -17,6 +17,7 @@ limitations under the License. package object import ( + "fmt" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -95,3 +96,89 @@ func TestGenerateCephXUser(t *testing.T) { fakeUser := generateCephXUser("rook-ceph-rgw-fake-store-fake-user") assert.Equal(t, "client.rgw.fake.store.fake.user", fakeUser) } + +// general testing theory here is that this easy unit test covers almost all of the RGW configs +// applied via mon store for all the various user input configurations. e2e tests will verify that +// RGW configs determined here are applied to the running RGW +func Test_clusterConfig_generateMonConfigOptions(t *testing.T) { + defaultConfigs := map[string]string{ + "rgw_enable_usage_log": "true", + "rgw_log_nonexistent_bucket": "true", + "rgw_log_object_name_utc": "true", + "rgw_run_sync_thread": "true", + "rgw_zone": "zone", + "rgw_zonegroup": "zone-group", + } + + // overlay string slice as map KVs on top of default configs (can append or modify) + overlayOnDefaultConfigs := func(kv ...string) map[string]string { + out := map[string]string{} + for k, v := range defaultConfigs { + out[k] = v + } + for i := 0; i < len(kv); i += 2 { + out[kv[i]] = kv[i+1] + } + return out + } + + tests := []struct { + name string // test name + objectStoreSpec *cephv1.ObjectStoreSpec + want map[string]string + wantErr bool + }{ + {"empty spec", &cephv1.ObjectStoreSpec{}, defaultConfigs, false}, + {"multisite sync enabled", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{DisableMultisiteSyncTraffic: true}, + }, overlayOnDefaultConfigs("rgw_run_sync_thread", "false"), false}, + {"empty rgwConfig", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{}}, + }, defaultConfigs, false}, + {"one add rgwConfig", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add"}}, + }, overlayOnDefaultConfigs("one", "add"), false}, + {"two add rgwConfig", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add", "two": "add"}}, + }, overlayOnDefaultConfigs("one", "add", "two", "add"), false}, + {"one add one modify rgwConfig", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add", "rgw_enable_usage_log": "false"}}, + }, overlayOnDefaultConfigs("one", "add", "rgw_enable_usage_log", "false"), false}, + {"rgwCommandFlags set", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwCommandFlags: map[string]string{"one": "add", "rgw_enable_usage_log": "false"}}, + }, defaultConfigs, false}, // verifies rgwCommandFlags don't affect mon config store + {"test all configs", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{ + DisableMultisiteSyncTraffic: true, + RgwConfig: map[string]string{"one": "add", "rgw_enable_usage_log": "false"}, + RgwCommandFlags: map[string]string{"two": "add", "rgw_zone": "bob"}}, + }, overlayOnDefaultConfigs("rgw_run_sync_thread", "false", "one", "add", "rgw_enable_usage_log", "false"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cos := &cephv1.CephObjectStore{ + Spec: *tt.objectStoreSpec, + } + cos.Namespace = "ns" + cos.Name = "my-store" + rgwConfig := &rgwConfig{ + ResourceName: "rook-ceph-rgw-my-store-a", + DaemonID: "my-store-a", + Realm: "realm", + ZoneGroup: "zone-group", + Zone: "zone", + } + + c := &clusterConfig{ + store: cos, + } + + got, err := c.generateMonConfigOptions(rgwConfig) + if (err != nil) != tt.wantErr { + t.Errorf("clusterConfig.generateMonConfigOptions() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, fmt.Sprint(tt.want), fmt.Sprint(got)) // go maps are stringified in sorted order for testing + }) + } +} diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 2d0ddaae1..e5930692b 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -88,34 +88,6 @@ func TestStartRGW(t *testing.T) { validateStart(ctx, t, c, clientset) }) - - t.Run("Multisite sync traffic disabled", func(t *testing.T) { - store.ObjectMeta.Name = "client-rgw-sync-disabled" - store.Spec.Gateway.DisableMultisiteSyncTraffic = true - - // Purge store of configurations applied to gateways - appliedRgwConfigurations = make(map[string]string) - - err := c.startRGWPods(store.Name, store.Name, store.Name, nil) - assert.Nil(t, err) - - assert.Contains(t, appliedRgwConfigurations, "rgw_run_sync_thread") - assert.Equal(t, appliedRgwConfigurations["rgw_run_sync_thread"], "false") - }) - - t.Run("Multisite sync traffic enabled", func(t *testing.T) { - store.ObjectMeta.Name = "client-rgw-sync-enabled" - store.Spec.Gateway.DisableMultisiteSyncTraffic = false - - // Purge store of configurations applied to gateways - appliedRgwConfigurations = make(map[string]string) - - err := c.startRGWPods(store.Name, store.Name, store.Name, nil) - assert.Nil(t, err) - - assert.Contains(t, appliedRgwConfigurations, "rgw_run_sync_thread") - assert.Equal(t, appliedRgwConfigurations["rgw_run_sync_thread"], "true") - }) } func validateStart(ctx context.Context, t *testing.T, c *clusterConfig, clientset *fclient.Clientset) { diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index b84b44b17..e3f5e4f06 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -437,6 +437,12 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, if hostingOptions != "" { container.Args = append(container.Args, hostingOptions) } + + // user configs are very last arguments so that they override what Rook might be setting before + for flag, val := range c.store.Spec.Gateway.RgwCommandFlags { + container.Args = append(container.Args, cephconfig.NewFlag(flag, val)) + } + return container, nil } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 32f65f618..548e631da 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -908,6 +908,87 @@ func TestAWSServerSideEncryption(t *testing.T) { }) } +func TestRgwCommandFlags(t *testing.T) { + //ctx := context.TODO() + // Placeholder + context := &clusterd.Context{Clientset: test.New(t, 3)} + + store := simpleStore() + info := clienttest.CreateTestClusterInfo(1) + info.CephVersion = cephver.CephVersion{Major: 18, Minor: 2, Extra: 0} + info.Namespace = store.Namespace + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default", "rook-ceph", "/var/lib/rook/") + + c := &clusterConfig{ + clusterInfo: info, + store: store, + context: context, + rookVersion: "rook/rook:myversion", + clusterSpec: &cephv1.ClusterSpec{ + CephVersion: cephv1.CephVersionSpec{Image: "quay.io/ceph/ceph:v19.3"}, + Network: cephv1.NetworkSpec{ + HostNetwork: true, + }, + }, + DataPathMap: data, + } + + resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name) + rgwConfig := &rgwConfig{ + ResourceName: resourceName, + DaemonID: "default", + } + + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + baseArgs := rgwContainer.Args // args without rgwCommandFlags set + + t.Run("empty map adds no flags", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{} + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + assert.Equal(t, baseArgs, rgwContainer.Args) + }) + + t.Run("add flag with space", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw option": "val spaced"} + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + l := len(rgwContainer.Args) + assert.Equal(t, "--rgw-option=val spaced", rgwContainer.Args[l-1]) + }) + + t.Run("add flag with dashes", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw-option": "val-dashed"} + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + l := len(rgwContainer.Args) + assert.Equal(t, "--rgw-option=val-dashed", rgwContainer.Args[l-1]) + }) + + t.Run("add flag with underscores", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw_option": "val_underscored"} + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + l := len(rgwContainer.Args) + assert.Equal(t, "--rgw-option=val_underscored", rgwContainer.Args[l-1]) + }) + + t.Run("add all flags", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{ + "rgw option": "val spaced", + "rgw-option": "val-dashed", + "rgw_option": "val_underscored", + } + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + l := len(rgwContainer.Args) + assert.Equal(t, "--rgw-option=val spaced", rgwContainer.Args[l-3]) + assert.Equal(t, "--rgw-option=val-dashed", rgwContainer.Args[l-2]) + assert.Equal(t, "--rgw-option=val_underscored", rgwContainer.Args[l-1]) + }) +} + func TestAddDNSNamesToRGWPodSpec(t *testing.T) { setupTest := func(zoneName string, cephvers cephver.CephVersion, dnsNames, customEndpoints []string) *clusterConfig { store := simpleStore()