diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index f8569de11..ec1f60a4f 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -349,6 +349,64 @@ vault write -f transit/keys/ exportable=true # transit engine * `tokenSecretName` can be (and often will be) the same for both kms and s3 configurations. +## Advanced configuration + +!!! warning + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + +CephObjectStore allows arbitrary Ceph configurations to be applied to RGW daemons that serve the +object store. [RGW config reference](https://docs.ceph.com/en/latest/radosgw/config-ref/). + +Configurations are applied to all RGWs that serve the CephObjectStore. Values must be strings. +Below is an example showing how different RGW configs and values might be applied. The example is +intended only to show a selection of value data types. + +```yaml +# THIS SAMPLE IS NOT A RECOMMENDATION +# ... +spec: + gateway: + # ... + rgwConfig: + debug_rgw: "10" # int + # debug-rgw: "20" # equivalent config keys can have dashes or underscores + rgw_s3_auth_use_ldap: "true" # bool + rgwCommandFlags: + rgw_dmclock_auth_res: "100.0" # float + rgw_d4n_l1_datacache_persistent_path: /var/log/rook/rgwd4ncache # string + rgw_d4n_address: "127.0.0.1:6379" # IP string +``` + +* `rgwConfig` - These configurations are applied and modified at runtime, without RGW restart. +* `rgwCommandFlags` - These configurations are applied as CLI arguments and result in RGW daemons + restarting when updates are applied. Restarts are desired behavior for some RGW configs. + +!!! note + Once an `rgwConfig` is set, it will not be removed from Ceph's central config store when removed + from the `rgwConfig` spec. Be sure to specifically set values back to their defaults once done. + With this in mind, `rgwCommandFlags` may be a better choice for temporary config values like + debug levels. + +### Example - debugging + +Users are often asked to provide RGW logs at a high log level when troubleshooting complex issues. +Apply log levels to RGWs easily using `rgwCommandFlags`. + +This spec will restart the RGW(s) with the highest level debugging enabled. + +```yaml +# ... +spec: + gateway: + # ... + rgwCommandFlags: + debug_ms: "20" + debug_rgw: "20" +``` + +Once RGW debug logging is no longer needed, the values can simply be removed from the spec. + ## Deleting a CephObjectStore During deletion of a CephObjectStore resource, Rook protects against accidental or premature diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 9530f3501..c086097bf 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7172,6 +7172,36 @@ Example: for an additional mount at subPath ldap, mounted from a se bindpass.secret, the file would reside at /var/rgw/ldap/bindpass.secret.

+ + +rgwConfig
+ +map[string]string + + + +(Optional) +

RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store. +Values are modified at runtime without RGW restart. +This feature is intended for advanced users. It allows breaking configurations to be easily +applied. Use with caution.

+ + + + +rgwCommandFlags
+ +map[string]string + + + +(Optional) +

RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object +store. Values are modified at RGW startup, resulting in RGW pod restarts. +This feature is intended for advanced users. It allows breaking configurations to be easily +applied. Use with caution.

+ +

HTTPEndpointSpec diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 37d9ed907..156fee79f 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -11869,6 +11869,26 @@ spec: type: object type: object x-kubernetes-preserve-unknown-fields: true + rgwCommandFlags: + additionalProperties: + type: string + description: |- + RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object + store. Values are modified at RGW startup, resulting in RGW pod restarts. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object + rgwConfig: + additionalProperties: + type: string + description: |- + RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store. + Values are modified at runtime without RGW restart. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object securePort: description: The port the rgw service will be listening on (https) format: int32 diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 67a17ec1a..3f557e0c8 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -11860,6 +11860,26 @@ spec: type: object type: object x-kubernetes-preserve-unknown-fields: true + rgwCommandFlags: + additionalProperties: + type: string + description: |- + RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object + store. Values are modified at RGW startup, resulting in RGW pod restarts. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object + rgwConfig: + additionalProperties: + type: string + description: |- + RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store. + Values are modified at runtime without RGW restart. + This feature is intended for advanced users. It allows breaking configurations to be easily + applied. Use with caution. + nullable: true + type: object securePort: description: The port the rgw service will be listening on (https) format: int32 diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 2ef5e2858..2f5804082 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1734,6 +1734,22 @@ type GatewaySpec struct { // Example: for an additional mount at subPath `ldap`, mounted from a secret that has key // `bindpass.secret`, the file would reside at `/var/rgw/ldap/bindpass.secret`. AdditionalVolumeMounts AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"` + + // RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store. + // Values are modified at runtime without RGW restart. + // This feature is intended for advanced users. It allows breaking configurations to be easily + // applied. Use with caution. + // +nullable + // +optional + RgwConfig map[string]string `json:"rgwConfig,omitempty"` + + // RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object + // store. Values are modified at RGW startup, resulting in RGW pod restarts. + // This feature is intended for advanced users. It allows breaking configurations to be easily + // applied. Use with caution. + // +nullable + // +optional + RgwCommandFlags map[string]string `json:"rgwCommandFlags,omitempty"` } // EndpointAddress is a tuple that describes a single IP address or host name. This is a subset of diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index b63b72756..38f4fbd25 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -2731,6 +2731,20 @@ func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.RgwConfig != nil { + in, out := &in.RgwConfig, &out.RgwConfig + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.RgwCommandFlags != nil { + in, out := &in.RgwCommandFlags, &out.RgwCommandFlags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 983589d0d..682aa5d2d 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -156,6 +156,20 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { monStore := cephconfig.GetMonStore(c.context, c.clusterInfo) who := generateCephXUser(rgwConfig.ResourceName) + + configOptions, err := c.generateMonConfigOptions(rgwConfig) + if err != nil { + return err + } + + if err := monStore.SetAll(who, configOptions); err != nil { + return errors.Wrapf(err, "failed to set all RGW configs on %q", who) + } + + return nil +} + +func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[string]string, error) { configOptions := make(map[string]string) configOptions["rgw_run_sync_thread"] = "true" @@ -171,7 +185,7 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { configOptions, err := configureKeystoneAuthentication(rgwConfig, configOptions) if err != nil { - return err + return configOptions, err } if s3 := rgwConfig.Protocols.S3; s3 != nil { @@ -192,14 +206,16 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { } } - for flag, val := range configOptions { - err := monStore.Set(who, flag, val) - if err != nil { - return errors.Wrapf(err, "failed to set %q to %q on %q", flag, val, who) + for flag, val := range c.store.Spec.Gateway.RgwConfig { + if currVal, ok := configOptions[flag]; ok { + // RGW might break with some user-specified config overrides; log clearly to help triage + logger.Infof("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q", + fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, val) } + configOptions[flag] = val } - return nil + return configOptions, nil } func configureKeystoneAuthentication(rgwConfig *rgwConfig, configOptions map[string]string) (map[string]string, error) { diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go index 88d3ad9d8..d0ba361c2 100644 --- a/pkg/operator/ceph/object/config_test.go +++ b/pkg/operator/ceph/object/config_test.go @@ -17,6 +17,7 @@ limitations under the License. package object import ( + "fmt" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -95,3 +96,89 @@ func TestGenerateCephXUser(t *testing.T) { fakeUser := generateCephXUser("rook-ceph-rgw-fake-store-fake-user") assert.Equal(t, "client.rgw.fake.store.fake.user", fakeUser) } + +// general testing theory here is that this easy unit test covers almost all of the RGW configs +// applied via mon store for all the various user input configurations. e2e tests will verify that +// RGW configs determined here are applied to the running RGW +func Test_clusterConfig_generateMonConfigOptions(t *testing.T) { + defaultConfigs := map[string]string{ + "rgw_enable_usage_log": "true", + "rgw_log_nonexistent_bucket": "true", + "rgw_log_object_name_utc": "true", + "rgw_run_sync_thread": "true", + "rgw_zone": "zone", + "rgw_zonegroup": "zone-group", + } + + // overlay string slice as map KVs on top of default configs (can append or modify) + overlayOnDefaultConfigs := func(kv ...string) map[string]string { + out := map[string]string{} + for k, v := range defaultConfigs { + out[k] = v + } + for i := 0; i < len(kv); i += 2 { + out[kv[i]] = kv[i+1] + } + return out + } + + tests := []struct { + name string // test name + objectStoreSpec *cephv1.ObjectStoreSpec + want map[string]string + wantErr bool + }{ + {"empty spec", &cephv1.ObjectStoreSpec{}, defaultConfigs, false}, + {"multisite sync enabled", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{DisableMultisiteSyncTraffic: true}, + }, overlayOnDefaultConfigs("rgw_run_sync_thread", "false"), false}, + {"empty rgwConfig", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{}}, + }, defaultConfigs, false}, + {"one add rgwConfig", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add"}}, + }, overlayOnDefaultConfigs("one", "add"), false}, + {"two add rgwConfig", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add", "two": "add"}}, + }, overlayOnDefaultConfigs("one", "add", "two", "add"), false}, + {"one add one modify rgwConfig", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add", "rgw_enable_usage_log": "false"}}, + }, overlayOnDefaultConfigs("one", "add", "rgw_enable_usage_log", "false"), false}, + {"rgwCommandFlags set", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{RgwCommandFlags: map[string]string{"one": "add", "rgw_enable_usage_log": "false"}}, + }, defaultConfigs, false}, // verifies rgwCommandFlags don't affect mon config store + {"test all configs", &cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{ + DisableMultisiteSyncTraffic: true, + RgwConfig: map[string]string{"one": "add", "rgw_enable_usage_log": "false"}, + RgwCommandFlags: map[string]string{"two": "add", "rgw_zone": "bob"}}, + }, overlayOnDefaultConfigs("rgw_run_sync_thread", "false", "one", "add", "rgw_enable_usage_log", "false"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cos := &cephv1.CephObjectStore{ + Spec: *tt.objectStoreSpec, + } + cos.Namespace = "ns" + cos.Name = "my-store" + rgwConfig := &rgwConfig{ + ResourceName: "rook-ceph-rgw-my-store-a", + DaemonID: "my-store-a", + Realm: "realm", + ZoneGroup: "zone-group", + Zone: "zone", + } + + c := &clusterConfig{ + store: cos, + } + + got, err := c.generateMonConfigOptions(rgwConfig) + if (err != nil) != tt.wantErr { + t.Errorf("clusterConfig.generateMonConfigOptions() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, fmt.Sprint(tt.want), fmt.Sprint(got)) // go maps are stringified in sorted order for testing + }) + } +} diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 2d0ddaae1..e5930692b 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -88,34 +88,6 @@ func TestStartRGW(t *testing.T) { validateStart(ctx, t, c, clientset) }) - - t.Run("Multisite sync traffic disabled", func(t *testing.T) { - store.ObjectMeta.Name = "client-rgw-sync-disabled" - store.Spec.Gateway.DisableMultisiteSyncTraffic = true - - // Purge store of configurations applied to gateways - appliedRgwConfigurations = make(map[string]string) - - err := c.startRGWPods(store.Name, store.Name, store.Name, nil) - assert.Nil(t, err) - - assert.Contains(t, appliedRgwConfigurations, "rgw_run_sync_thread") - assert.Equal(t, appliedRgwConfigurations["rgw_run_sync_thread"], "false") - }) - - t.Run("Multisite sync traffic enabled", func(t *testing.T) { - store.ObjectMeta.Name = "client-rgw-sync-enabled" - store.Spec.Gateway.DisableMultisiteSyncTraffic = false - - // Purge store of configurations applied to gateways - appliedRgwConfigurations = make(map[string]string) - - err := c.startRGWPods(store.Name, store.Name, store.Name, nil) - assert.Nil(t, err) - - assert.Contains(t, appliedRgwConfigurations, "rgw_run_sync_thread") - assert.Equal(t, appliedRgwConfigurations["rgw_run_sync_thread"], "true") - }) } func validateStart(ctx context.Context, t *testing.T, c *clusterConfig, clientset *fclient.Clientset) { diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index b84b44b17..e3f5e4f06 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -437,6 +437,12 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, if hostingOptions != "" { container.Args = append(container.Args, hostingOptions) } + + // user configs are very last arguments so that they override what Rook might be setting before + for flag, val := range c.store.Spec.Gateway.RgwCommandFlags { + container.Args = append(container.Args, cephconfig.NewFlag(flag, val)) + } + return container, nil } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 32f65f618..548e631da 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -908,6 +908,87 @@ func TestAWSServerSideEncryption(t *testing.T) { }) } +func TestRgwCommandFlags(t *testing.T) { + //ctx := context.TODO() + // Placeholder + context := &clusterd.Context{Clientset: test.New(t, 3)} + + store := simpleStore() + info := clienttest.CreateTestClusterInfo(1) + info.CephVersion = cephver.CephVersion{Major: 18, Minor: 2, Extra: 0} + info.Namespace = store.Namespace + data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default", "rook-ceph", "/var/lib/rook/") + + c := &clusterConfig{ + clusterInfo: info, + store: store, + context: context, + rookVersion: "rook/rook:myversion", + clusterSpec: &cephv1.ClusterSpec{ + CephVersion: cephv1.CephVersionSpec{Image: "quay.io/ceph/ceph:v19.3"}, + Network: cephv1.NetworkSpec{ + HostNetwork: true, + }, + }, + DataPathMap: data, + } + + resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name) + rgwConfig := &rgwConfig{ + ResourceName: resourceName, + DaemonID: "default", + } + + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + baseArgs := rgwContainer.Args // args without rgwCommandFlags set + + t.Run("empty map adds no flags", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{} + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + assert.Equal(t, baseArgs, rgwContainer.Args) + }) + + t.Run("add flag with space", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw option": "val spaced"} + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + l := len(rgwContainer.Args) + assert.Equal(t, "--rgw-option=val spaced", rgwContainer.Args[l-1]) + }) + + t.Run("add flag with dashes", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw-option": "val-dashed"} + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + l := len(rgwContainer.Args) + assert.Equal(t, "--rgw-option=val-dashed", rgwContainer.Args[l-1]) + }) + + t.Run("add flag with underscores", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw_option": "val_underscored"} + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + l := len(rgwContainer.Args) + assert.Equal(t, "--rgw-option=val_underscored", rgwContainer.Args[l-1]) + }) + + t.Run("add all flags", func(t *testing.T) { + c.store.Spec.Gateway.RgwCommandFlags = map[string]string{ + "rgw option": "val spaced", + "rgw-option": "val-dashed", + "rgw_option": "val_underscored", + } + rgwContainer, err := c.makeDaemonContainer(rgwConfig) + assert.NoError(t, err) + l := len(rgwContainer.Args) + assert.Equal(t, "--rgw-option=val spaced", rgwContainer.Args[l-3]) + assert.Equal(t, "--rgw-option=val-dashed", rgwContainer.Args[l-2]) + assert.Equal(t, "--rgw-option=val_underscored", rgwContainer.Args[l-1]) + }) +} + func TestAddDNSNamesToRGWPodSpec(t *testing.T) { setupTest := func(zoneName string, cephvers cephver.CephVersion, dnsNames, customEndpoints []string) *clusterConfig { store := simpleStore()