diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
index f8569de11..ec1f60a4f 100644
--- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
+++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
@@ -349,6 +349,64 @@ vault write -f transit/keys/ exportable=true # transit engine
* `tokenSecretName` can be (and often will be) the same for both kms and s3 configurations.
+## Advanced configuration
+
+!!! warning
+ This feature is intended for advanced users. It allows breaking configurations to be easily
+ applied. Use with caution.
+
+CephObjectStore allows arbitrary Ceph configurations to be applied to RGW daemons that serve the
+object store. [RGW config reference](https://docs.ceph.com/en/latest/radosgw/config-ref/).
+
+Configurations are applied to all RGWs that serve the CephObjectStore. Values must be strings.
+Below is an example showing how different RGW configs and values might be applied. The example is
+intended only to show a selection of value data types.
+
+```yaml
+# THIS SAMPLE IS NOT A RECOMMENDATION
+# ...
+spec:
+ gateway:
+ # ...
+ rgwConfig:
+ debug_rgw: "10" # int
+ # debug-rgw: "20" # equivalent config keys can have dashes or underscores
+ rgw_s3_auth_use_ldap: "true" # bool
+ rgwCommandFlags:
+ rgw_dmclock_auth_res: "100.0" # float
+ rgw_d4n_l1_datacache_persistent_path: /var/log/rook/rgwd4ncache # string
+ rgw_d4n_address: "127.0.0.1:6379" # IP string
+```
+
+* `rgwConfig` - These configurations are applied and modified at runtime, without RGW restart.
+* `rgwCommandFlags` - These configurations are applied as CLI arguments and result in RGW daemons
+ restarting when updates are applied. Restarts are desired behavior for some RGW configs.
+
+!!! note
+ Once an `rgwConfig` is set, it will not be removed from Ceph's central config store when removed
+ from the `rgwConfig` spec. Be sure to specifically set values back to their defaults once done.
+ With this in mind, `rgwCommandFlags` may be a better choice for temporary config values like
+ debug levels.
+
+### Example - debugging
+
+Users are often asked to provide RGW logs at a high log level when troubleshooting complex issues.
+Apply log levels to RGWs easily using `rgwCommandFlags`.
+
+This spec will restart the RGW(s) with the highest level debugging enabled.
+
+```yaml
+# ...
+spec:
+ gateway:
+ # ...
+ rgwCommandFlags:
+ debug_ms: "20"
+ debug_rgw: "20"
+```
+
+Once RGW debug logging is no longer needed, the values can simply be removed from the spec.
+
## Deleting a CephObjectStore
During deletion of a CephObjectStore resource, Rook protects against accidental or premature
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index 9530f3501..c086097bf 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -7172,6 +7172,36 @@ Example: for an additional mount at subPath ldap, mounted from a se
bindpass.secret, the file would reside at /var/rgw/ldap/bindpass.secret.
+
+
+rgwConfig
+
+map[string]string
+
+ |
+
+(Optional)
+ RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store.
+Values are modified at runtime without RGW restart.
+This feature is intended for advanced users. It allows breaking configurations to be easily
+applied. Use with caution.
+ |
+
+
+
+rgwCommandFlags
+
+map[string]string
+
+ |
+
+(Optional)
+ RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object
+store. Values are modified at RGW startup, resulting in RGW pod restarts.
+This feature is intended for advanced users. It allows breaking configurations to be easily
+applied. Use with caution.
+ |
+
HTTPEndpointSpec
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index 37d9ed907..156fee79f 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -11869,6 +11869,26 @@ spec:
type: object
type: object
x-kubernetes-preserve-unknown-fields: true
+ rgwCommandFlags:
+ additionalProperties:
+ type: string
+ description: |-
+ RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object
+ store. Values are modified at RGW startup, resulting in RGW pod restarts.
+ This feature is intended for advanced users. It allows breaking configurations to be easily
+ applied. Use with caution.
+ nullable: true
+ type: object
+ rgwConfig:
+ additionalProperties:
+ type: string
+ description: |-
+ RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store.
+ Values are modified at runtime without RGW restart.
+ This feature is intended for advanced users. It allows breaking configurations to be easily
+ applied. Use with caution.
+ nullable: true
+ type: object
securePort:
description: The port the rgw service will be listening on (https)
format: int32
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 67a17ec1a..3f557e0c8 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -11860,6 +11860,26 @@ spec:
type: object
type: object
x-kubernetes-preserve-unknown-fields: true
+ rgwCommandFlags:
+ additionalProperties:
+ type: string
+ description: |-
+ RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object
+ store. Values are modified at RGW startup, resulting in RGW pod restarts.
+ This feature is intended for advanced users. It allows breaking configurations to be easily
+ applied. Use with caution.
+ nullable: true
+ type: object
+ rgwConfig:
+ additionalProperties:
+ type: string
+ description: |-
+ RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store.
+ Values are modified at runtime without RGW restart.
+ This feature is intended for advanced users. It allows breaking configurations to be easily
+ applied. Use with caution.
+ nullable: true
+ type: object
securePort:
description: The port the rgw service will be listening on (https)
format: int32
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 2ef5e2858..2f5804082 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -1734,6 +1734,22 @@ type GatewaySpec struct {
// Example: for an additional mount at subPath `ldap`, mounted from a secret that has key
// `bindpass.secret`, the file would reside at `/var/rgw/ldap/bindpass.secret`.
AdditionalVolumeMounts AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
+
+ // RgwConfig sets Ceph RGW config values for the gateway clients that serve this object store.
+ // Values are modified at runtime without RGW restart.
+ // This feature is intended for advanced users. It allows breaking configurations to be easily
+ // applied. Use with caution.
+ // +nullable
+ // +optional
+ RgwConfig map[string]string `json:"rgwConfig,omitempty"`
+
+ // RgwCommandFlags sets Ceph RGW config values for the gateway clients that serve this object
+ // store. Values are modified at RGW startup, resulting in RGW pod restarts.
+ // This feature is intended for advanced users. It allows breaking configurations to be easily
+ // applied. Use with caution.
+ // +nullable
+ // +optional
+ RgwCommandFlags map[string]string `json:"rgwCommandFlags,omitempty"`
}
// EndpointAddress is a tuple that describes a single IP address or host name. This is a subset of
diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
index b63b72756..38f4fbd25 100644
--- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
@@ -2731,6 +2731,20 @@ func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
+ if in.RgwConfig != nil {
+ in, out := &in.RgwConfig, &out.RgwConfig
+ *out = make(map[string]string, len(*in))
+ for key, val := range *in {
+ (*out)[key] = val
+ }
+ }
+ if in.RgwCommandFlags != nil {
+ in, out := &in.RgwCommandFlags, &out.RgwCommandFlags
+ *out = make(map[string]string, len(*in))
+ for key, val := range *in {
+ (*out)[key] = val
+ }
+ }
return
}
diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go
index 983589d0d..682aa5d2d 100644
--- a/pkg/operator/ceph/object/config.go
+++ b/pkg/operator/ceph/object/config.go
@@ -156,6 +156,20 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error {
monStore := cephconfig.GetMonStore(c.context, c.clusterInfo)
who := generateCephXUser(rgwConfig.ResourceName)
+
+ configOptions, err := c.generateMonConfigOptions(rgwConfig)
+ if err != nil {
+ return err
+ }
+
+ if err := monStore.SetAll(who, configOptions); err != nil {
+ return errors.Wrapf(err, "failed to set all RGW configs on %q", who)
+ }
+
+ return nil
+}
+
+func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[string]string, error) {
configOptions := make(map[string]string)
configOptions["rgw_run_sync_thread"] = "true"
@@ -171,7 +185,7 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error {
configOptions, err := configureKeystoneAuthentication(rgwConfig, configOptions)
if err != nil {
- return err
+ return configOptions, err
}
if s3 := rgwConfig.Protocols.S3; s3 != nil {
@@ -192,14 +206,16 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error {
}
}
- for flag, val := range configOptions {
- err := monStore.Set(who, flag, val)
- if err != nil {
- return errors.Wrapf(err, "failed to set %q to %q on %q", flag, val, who)
+ for flag, val := range c.store.Spec.Gateway.RgwConfig {
+ if currVal, ok := configOptions[flag]; ok {
+ // RGW might break with some user-specified config overrides; log clearly to help triage
+ logger.Infof("overriding object store %q RGW config option %q (current value %q) with user-specified rgwConfig %q",
+ fmt.Sprintf("%s/%s", c.store.Namespace, c.store.Name), flag, currVal, val)
}
+ configOptions[flag] = val
}
- return nil
+ return configOptions, nil
}
func configureKeystoneAuthentication(rgwConfig *rgwConfig, configOptions map[string]string) (map[string]string, error) {
diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go
index 88d3ad9d8..d0ba361c2 100644
--- a/pkg/operator/ceph/object/config_test.go
+++ b/pkg/operator/ceph/object/config_test.go
@@ -17,6 +17,7 @@ limitations under the License.
package object
import (
+ "fmt"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -95,3 +96,89 @@ func TestGenerateCephXUser(t *testing.T) {
fakeUser := generateCephXUser("rook-ceph-rgw-fake-store-fake-user")
assert.Equal(t, "client.rgw.fake.store.fake.user", fakeUser)
}
+
+// general testing theory here is that this easy unit test covers almost all of the RGW configs
+// applied via mon store for all the various user input configurations. e2e tests will verify that
+// RGW configs determined here are applied to the running RGW
+func Test_clusterConfig_generateMonConfigOptions(t *testing.T) {
+ defaultConfigs := map[string]string{
+ "rgw_enable_usage_log": "true",
+ "rgw_log_nonexistent_bucket": "true",
+ "rgw_log_object_name_utc": "true",
+ "rgw_run_sync_thread": "true",
+ "rgw_zone": "zone",
+ "rgw_zonegroup": "zone-group",
+ }
+
+ // overlay string slice as map KVs on top of default configs (can append or modify)
+ overlayOnDefaultConfigs := func(kv ...string) map[string]string {
+ out := map[string]string{}
+ for k, v := range defaultConfigs {
+ out[k] = v
+ }
+ for i := 0; i < len(kv); i += 2 {
+ out[kv[i]] = kv[i+1]
+ }
+ return out
+ }
+
+ tests := []struct {
+ name string // test name
+ objectStoreSpec *cephv1.ObjectStoreSpec
+ want map[string]string
+ wantErr bool
+ }{
+ {"empty spec", &cephv1.ObjectStoreSpec{}, defaultConfigs, false},
+ {"multisite sync enabled", &cephv1.ObjectStoreSpec{
+ Gateway: cephv1.GatewaySpec{DisableMultisiteSyncTraffic: true},
+ }, overlayOnDefaultConfigs("rgw_run_sync_thread", "false"), false},
+ {"empty rgwConfig", &cephv1.ObjectStoreSpec{
+ Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{}},
+ }, defaultConfigs, false},
+ {"one add rgwConfig", &cephv1.ObjectStoreSpec{
+ Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add"}},
+ }, overlayOnDefaultConfigs("one", "add"), false},
+ {"two add rgwConfig", &cephv1.ObjectStoreSpec{
+ Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add", "two": "add"}},
+ }, overlayOnDefaultConfigs("one", "add", "two", "add"), false},
+ {"one add one modify rgwConfig", &cephv1.ObjectStoreSpec{
+ Gateway: cephv1.GatewaySpec{RgwConfig: map[string]string{"one": "add", "rgw_enable_usage_log": "false"}},
+ }, overlayOnDefaultConfigs("one", "add", "rgw_enable_usage_log", "false"), false},
+ {"rgwCommandFlags set", &cephv1.ObjectStoreSpec{
+ Gateway: cephv1.GatewaySpec{RgwCommandFlags: map[string]string{"one": "add", "rgw_enable_usage_log": "false"}},
+ }, defaultConfigs, false}, // verifies rgwCommandFlags don't affect mon config store
+ {"test all configs", &cephv1.ObjectStoreSpec{
+ Gateway: cephv1.GatewaySpec{
+ DisableMultisiteSyncTraffic: true,
+ RgwConfig: map[string]string{"one": "add", "rgw_enable_usage_log": "false"},
+ RgwCommandFlags: map[string]string{"two": "add", "rgw_zone": "bob"}},
+ }, overlayOnDefaultConfigs("rgw_run_sync_thread", "false", "one", "add", "rgw_enable_usage_log", "false"), false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ cos := &cephv1.CephObjectStore{
+ Spec: *tt.objectStoreSpec,
+ }
+ cos.Namespace = "ns"
+ cos.Name = "my-store"
+ rgwConfig := &rgwConfig{
+ ResourceName: "rook-ceph-rgw-my-store-a",
+ DaemonID: "my-store-a",
+ Realm: "realm",
+ ZoneGroup: "zone-group",
+ Zone: "zone",
+ }
+
+ c := &clusterConfig{
+ store: cos,
+ }
+
+ got, err := c.generateMonConfigOptions(rgwConfig)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("clusterConfig.generateMonConfigOptions() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ assert.Equal(t, fmt.Sprint(tt.want), fmt.Sprint(got)) // go maps are stringified in sorted order for testing
+ })
+ }
+}
diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go
index 2d0ddaae1..e5930692b 100644
--- a/pkg/operator/ceph/object/rgw_test.go
+++ b/pkg/operator/ceph/object/rgw_test.go
@@ -88,34 +88,6 @@ func TestStartRGW(t *testing.T) {
validateStart(ctx, t, c, clientset)
})
-
- t.Run("Multisite sync traffic disabled", func(t *testing.T) {
- store.ObjectMeta.Name = "client-rgw-sync-disabled"
- store.Spec.Gateway.DisableMultisiteSyncTraffic = true
-
- // Purge store of configurations applied to gateways
- appliedRgwConfigurations = make(map[string]string)
-
- err := c.startRGWPods(store.Name, store.Name, store.Name, nil)
- assert.Nil(t, err)
-
- assert.Contains(t, appliedRgwConfigurations, "rgw_run_sync_thread")
- assert.Equal(t, appliedRgwConfigurations["rgw_run_sync_thread"], "false")
- })
-
- t.Run("Multisite sync traffic enabled", func(t *testing.T) {
- store.ObjectMeta.Name = "client-rgw-sync-enabled"
- store.Spec.Gateway.DisableMultisiteSyncTraffic = false
-
- // Purge store of configurations applied to gateways
- appliedRgwConfigurations = make(map[string]string)
-
- err := c.startRGWPods(store.Name, store.Name, store.Name, nil)
- assert.Nil(t, err)
-
- assert.Contains(t, appliedRgwConfigurations, "rgw_run_sync_thread")
- assert.Equal(t, appliedRgwConfigurations["rgw_run_sync_thread"], "true")
- })
}
func validateStart(ctx context.Context, t *testing.T, c *clusterConfig, clientset *fclient.Clientset) {
diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go
index b84b44b17..e3f5e4f06 100644
--- a/pkg/operator/ceph/object/spec.go
+++ b/pkg/operator/ceph/object/spec.go
@@ -437,6 +437,12 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
if hostingOptions != "" {
container.Args = append(container.Args, hostingOptions)
}
+
+ // user configs are very last arguments so that they override what Rook might be setting before
+ for flag, val := range c.store.Spec.Gateway.RgwCommandFlags {
+ container.Args = append(container.Args, cephconfig.NewFlag(flag, val))
+ }
+
return container, nil
}
diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go
index 32f65f618..548e631da 100644
--- a/pkg/operator/ceph/object/spec_test.go
+++ b/pkg/operator/ceph/object/spec_test.go
@@ -908,6 +908,87 @@ func TestAWSServerSideEncryption(t *testing.T) {
})
}
+func TestRgwCommandFlags(t *testing.T) {
+ //ctx := context.TODO()
+ // Placeholder
+ context := &clusterd.Context{Clientset: test.New(t, 3)}
+
+ store := simpleStore()
+ info := clienttest.CreateTestClusterInfo(1)
+ info.CephVersion = cephver.CephVersion{Major: 18, Minor: 2, Extra: 0}
+ info.Namespace = store.Namespace
+ data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default", "rook-ceph", "/var/lib/rook/")
+
+ c := &clusterConfig{
+ clusterInfo: info,
+ store: store,
+ context: context,
+ rookVersion: "rook/rook:myversion",
+ clusterSpec: &cephv1.ClusterSpec{
+ CephVersion: cephv1.CephVersionSpec{Image: "quay.io/ceph/ceph:v19.3"},
+ Network: cephv1.NetworkSpec{
+ HostNetwork: true,
+ },
+ },
+ DataPathMap: data,
+ }
+
+ resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name)
+ rgwConfig := &rgwConfig{
+ ResourceName: resourceName,
+ DaemonID: "default",
+ }
+
+ rgwContainer, err := c.makeDaemonContainer(rgwConfig)
+ assert.NoError(t, err)
+ baseArgs := rgwContainer.Args // args without rgwCommandFlags set
+
+ t.Run("empty map adds no flags", func(t *testing.T) {
+ c.store.Spec.Gateway.RgwCommandFlags = map[string]string{}
+ rgwContainer, err := c.makeDaemonContainer(rgwConfig)
+ assert.NoError(t, err)
+ assert.Equal(t, baseArgs, rgwContainer.Args)
+ })
+
+ t.Run("add flag with space", func(t *testing.T) {
+ c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw option": "val spaced"}
+ rgwContainer, err := c.makeDaemonContainer(rgwConfig)
+ assert.NoError(t, err)
+ l := len(rgwContainer.Args)
+ assert.Equal(t, "--rgw-option=val spaced", rgwContainer.Args[l-1])
+ })
+
+ t.Run("add flag with dashes", func(t *testing.T) {
+ c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw-option": "val-dashed"}
+ rgwContainer, err := c.makeDaemonContainer(rgwConfig)
+ assert.NoError(t, err)
+ l := len(rgwContainer.Args)
+ assert.Equal(t, "--rgw-option=val-dashed", rgwContainer.Args[l-1])
+ })
+
+ t.Run("add flag with underscores", func(t *testing.T) {
+ c.store.Spec.Gateway.RgwCommandFlags = map[string]string{"rgw_option": "val_underscored"}
+ rgwContainer, err := c.makeDaemonContainer(rgwConfig)
+ assert.NoError(t, err)
+ l := len(rgwContainer.Args)
+ assert.Equal(t, "--rgw-option=val_underscored", rgwContainer.Args[l-1])
+ })
+
+ t.Run("add all flags", func(t *testing.T) {
+ c.store.Spec.Gateway.RgwCommandFlags = map[string]string{
+ "rgw option": "val spaced",
+ "rgw-option": "val-dashed",
+ "rgw_option": "val_underscored",
+ }
+ rgwContainer, err := c.makeDaemonContainer(rgwConfig)
+ assert.NoError(t, err)
+ l := len(rgwContainer.Args)
+ assert.Equal(t, "--rgw-option=val spaced", rgwContainer.Args[l-3])
+ assert.Equal(t, "--rgw-option=val-dashed", rgwContainer.Args[l-2])
+ assert.Equal(t, "--rgw-option=val_underscored", rgwContainer.Args[l-1])
+ })
+}
+
func TestAddDNSNamesToRGWPodSpec(t *testing.T) {
setupTest := func(zoneName string, cephvers cephver.CephVersion, dnsNames, customEndpoints []string) *clusterConfig {
store := simpleStore()