diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
index 82935cd93..79f4e5dd0 100755
--- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md
+++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
@@ -92,6 +92,7 @@ For more details on the mons and when to choose a number other than `3`, see the
* `removeOSDsIfOutAndSafeToRemove`: If `true` the operator will remove the OSDs that are down and whose data has been restored to other OSDs. In Ceph terms, the OSDs are `out` and `safe-to-destroy` when they are removed.
* `cleanupPolicy`: [cleanup policy settings](#cleanup-policy)
* `security`: [security page for key management configuration](../../Storage-Configuration/Advanced/key-management-system.md)
+* `cephConfig`: [Set Ceph config options using the Ceph Mon config store](#ceph-config)
### Ceph container images
@@ -937,3 +938,31 @@ However, all new configuration by the operator will be blocked with this cleanup
Rook waits for the deletion of PVs provisioned using the cephCluster before proceeding to delete the
cephCluster. To force deletion of the cephCluster without waiting for the PVs to be deleted, you can
set the `allowUninstallWithVolumes` to true under `spec.CleanupPolicy`.
+
+## Ceph Config
+
+!!! attention
+ This feature is experimental.
+
+The Ceph config options are applied after the MONs are all in quorum and running. To set Ceph config options, you can add them to your `CephCluster` spec like this:
+
+```yaml
+spec:
+ # [...]
+ cephConfig:
+ # Who's the target for these config options?
+ global:
+ # All values must be quoted so they are considered a string in YAML
+ osd_pool_default_size: "3"
+ mon_warn_on_pool_no_redundancy: "false"
+ osd_crush_update_on_start: "false"
+ # Make sure to quote special characters
+ "osd.*":
+ osd_max_scrubs: "10"
+```
+
+!!! warning
+ Rook performs no direct validation on these config options, so the validity of the settings is the
+ user's responsibility.
+
+The operator does not unset any removed config options, it is the user's responsibility to unset or set the default value for each removed option manually using the Ceph CLI.
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index f55b3c4b2..ddc9591eb 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -1113,6 +1113,18 @@ CSIDriverSpec
CSI Driver Options applied per cluster.
+
+
+cephConfig
+
+map[string]map[string]string
+
+ |
+
+(Optional)
+ Ceph Config options
+ |
+
@@ -4389,6 +4401,18 @@ CSIDriverSpec
CSI Driver Options applied per cluster.
+
+
+cephConfig
+
+map[string]map[string]string
+
+ |
+
+(Optional)
+ Ceph Config options
+ |
+
ClusterState
diff --git a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md
index 1ba87a4dd..92a0db0a2 100644
--- a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md
+++ b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md
@@ -206,11 +206,13 @@ ceph osd pool set rbd pg_num 512
## Custom `ceph.conf` Settings
!!! warning
- The advised method for controlling Ceph configuration is to manually use the Ceph CLI
- or the Ceph dashboard because this offers the most flexibility. It is highly recommended that this
- only be used when absolutely necessary and that the `config` be reset to an empty string if/when the
- configurations are no longer necessary. Configurations in the config file will make the Ceph cluster
- less configurable from the CLI and dashboard and may make future tuning or debugging difficult.
+ The advised method for controlling Ceph configuration is to use the [`cephConfig:` structure](../../CRDs/Cluster/ceph-cluster-crd.md#ceph-config)
+ in the `CephCluster` CRD.
+
+ It is highly recommended that this only be used when absolutely necessary and that the `config` be
+ reset to an empty string if/when the configurations are no longer necessary. Configurations in the
+ config file will make the Ceph cluster less configurable from the CLI and dashboard and may make
+ future tuning or debugging difficult.
Setting configs via Ceph's CLI requires that at least one mon be available for the configs to be
set, and setting configs via dashboard requires at least one mgr to be available. Ceph also has
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index 7ea0ea59a..00628c50f 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -7,3 +7,5 @@
`ROOK_DISABLE_ADMISSION_CONTROLLER: "true"` in operator.yaml before upgrading to rook v1.13. CRD validation is now enabled with [Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/). This requires Kubernetes version 1.25 or higher.
## Features
+
+- Added `cephConfig:` to CephCluster to allow setting Ceph config options in the Ceph MON config store via the CRD
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index 2d9a514c6..c92de4811 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -895,6 +895,14 @@ spec:
nullable: true
type: object
x-kubernetes-preserve-unknown-fields: true
+ cephConfig:
+ additionalProperties:
+ additionalProperties:
+ type: string
+ type: object
+ description: Ceph Config options
+ nullable: true
+ type: object
cephVersion:
description: The version information that instructs Rook to orchestrate a particular version of Ceph.
nullable: true
diff --git a/deploy/examples/cluster-multus-test.yaml b/deploy/examples/cluster-multus-test.yaml
index 3b11f8dd4..f0d87caef 100644
--- a/deploy/examples/cluster-multus-test.yaml
+++ b/deploy/examples/cluster-multus-test.yaml
@@ -7,19 +7,6 @@
# kubectl create -f crds.yaml -f common.yaml -f operator.yaml
# kubectl create -f cluster-test.yaml
#################################################################################################################
-kind: ConfigMap
-apiVersion: v1
-metadata:
- name: rook-config-override
- namespace: rook-ceph # namespace:cluster
-data:
- config: |
- [global]
- osd_pool_default_size = 1
- mon_warn_on_pool_no_redundancy = false
- bdev_flock_retry = 20
- bluefs_buffered_io = false
----
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
@@ -47,3 +34,10 @@ spec:
useAllNodes: true
useAllDevices: true
#deviceFilter:
+ cephConfig:
+ global:
+ osd_pool_default_size: "1"
+ mon_warn_on_pool_no_redundancy: "false"
+ bdev_flock_retry: "20"
+ bluefs_buffered_io: "false"
+ mon_data_avail_warn: "10"
diff --git a/deploy/examples/cluster-test.yaml b/deploy/examples/cluster-test.yaml
index 58c915451..4d25d0225 100644
--- a/deploy/examples/cluster-test.yaml
+++ b/deploy/examples/cluster-test.yaml
@@ -7,20 +7,6 @@
# kubectl create -f crds.yaml -f common.yaml -f operator.yaml
# kubectl create -f cluster-test.yaml
#################################################################################################################
-kind: ConfigMap
-apiVersion: v1
-metadata:
- name: rook-config-override
- namespace: rook-ceph # namespace:cluster
-data:
- config: |
- [global]
- osd_pool_default_size = 1
- mon_warn_on_pool_no_redundancy = false
- bdev_flock_retry = 20
- bluefs_buffered_io = false
- mon_data_avail_warn = 10
----
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
@@ -57,6 +43,13 @@ spec:
mgr: system-cluster-critical
disruptionManagement:
managePodBudgets: true
+ cephConfig:
+ global:
+ osd_pool_default_size: "1"
+ mon_warn_on_pool_no_redundancy: "false"
+ bdev_flock_retry: "20"
+ bluefs_buffered_io: "false"
+ mon_data_avail_warn: "10"
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 6e17fbb56..c747bb2bf 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -893,6 +893,14 @@ spec:
nullable: true
type: object
x-kubernetes-preserve-unknown-fields: true
+ cephConfig:
+ additionalProperties:
+ additionalProperties:
+ type: string
+ type: object
+ description: Ceph Config options
+ nullable: true
+ type: object
cephVersion:
description: The version information that instructs Rook to orchestrate a particular version of Ceph.
nullable: true
diff --git a/design/ceph/ceph-config-via-cluster-crd.md b/design/ceph/ceph-config-via-cluster-crd.md
index d4b3a9dd8..43f70eb93 100644
--- a/design/ceph/ceph-config-via-cluster-crd.md
+++ b/design/ceph/ceph-config-via-cluster-crd.md
@@ -51,3 +51,5 @@ The operator will use the Ceph config store (that is accessed via `ceph config a
### Limitations
The operator won't be unsetting any previously set config options or restore config options to their default value.
+
+The operator will ignore a few vital config options, such as `fsid`, `keyring` and `mon_host`.
diff --git a/go.mod b/go.mod
index a9f0bd6fc..79b66dbb7 100644
--- a/go.mod
+++ b/go.mod
@@ -28,6 +28,7 @@ require (
github.com/stretchr/testify v1.8.4
github.com/sykesm/zap-logfmt v0.0.4
go.uber.org/zap v1.26.0
+ golang.org/x/exp v0.0.0-20230206171751-46f607a40771
golang.org/x/sync v0.5.0
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v2 v2.4.0
@@ -116,7 +117,6 @@ require (
go.starlark.net v0.0.0-20230814145427-12f4cb8177e4 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
- golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sys v0.13.0 // indirect
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 2e6e3710f..ca5eaa797 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -226,6 +226,11 @@ type ClusterSpec struct {
// CSI Driver Options applied per cluster.
// +optional
CSI CSIDriverSpec `json:"csi,omitempty"`
+
+ // Ceph Config options
+ // +optional
+ // +nullable
+ CephConfig map[string]map[string]string `json:"cephConfig,omitempty"`
}
// CSIDriverSpec defines CSI Driver settings applied per cluster.
diff --git a/pkg/daemon/ceph/client/pool_test.go b/pkg/daemon/ceph/client/pool_test.go
index 7b101d14e..c3fa975c8 100644
--- a/pkg/daemon/ceph/client/pool_test.go
+++ b/pkg/daemon/ceph/client/pool_test.go
@@ -22,6 +22,8 @@ import (
"strconv"
"testing"
+ "golang.org/x/exp/slices"
+
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
@@ -214,7 +216,7 @@ func testCreateReplicaPool(t *testing.T, failureDomain, crushRoot, deviceClass,
assert.Equal(t, failureDomain, args[6])
}
if deviceClass == "" {
- assert.False(t, testIsStringInSlice("hdd", args))
+ assert.False(t, slices.Contains(args, "hdd"))
} else {
assert.Equal(t, deviceClass, args[7])
}
@@ -336,15 +338,6 @@ func TestExtractPoolDetails(t *testing.T) {
})
}
-func testIsStringInSlice(a string, list []string) bool {
- for _, b := range list {
- if b == a {
- return true
- }
- }
- return false
-}
-
func TestGetPoolStatistics(t *testing.T) {
p := PoolStatistics{}
p.Images.Count = 1
diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go
index b3ea61502..e3e64668d 100755
--- a/pkg/operator/ceph/cluster/cluster.go
+++ b/pkg/operator/ceph/cluster/cluster.go
@@ -458,20 +458,17 @@ func (c *cluster) preMonStartupActions(cephVersion cephver.CephVersion) error {
// Basically, it is executed between the monitors and the manager sequence
func (c *cluster) postMonStartupActions() error {
// Create CSI Kubernetes Secrets
- err := csi.CreateCSISecrets(c.context, c.ClusterInfo)
- if err != nil {
+ if err := csi.CreateCSISecrets(c.context, c.ClusterInfo); err != nil {
return errors.Wrap(err, "failed to create csi kubernetes secrets")
}
// Create crash collector Kubernetes Secret
- err = nodedaemon.CreateCrashCollectorSecret(c.context, c.ClusterInfo)
- if err != nil {
+ if err := nodedaemon.CreateCrashCollectorSecret(c.context, c.ClusterInfo); err != nil {
return errors.Wrap(err, "failed to create crash collector kubernetes secret")
}
// Create exporter Kubernetes Secret
- err = nodedaemon.CreateExporterSecret(c.context, c.ClusterInfo)
- if err != nil {
+ if err := nodedaemon.CreateExporterSecret(c.context, c.ClusterInfo); err != nil {
return errors.Wrap(err, "failed to create exporter kubernetes secret")
}
@@ -479,6 +476,11 @@ func (c *cluster) postMonStartupActions() error {
return errors.Wrap(err, "failed to configure msgr2")
}
+ // Set config store options
+ if err := c.updateConfigStoreFromCRD(); err != nil {
+ return errors.Wrap(err, "")
+ }
+
crushRoot := client.GetCrushRootFromSpec(c.Spec)
if crushRoot != "default" {
// Remove the root=default and replicated_rule which are created by
@@ -490,14 +492,18 @@ func (c *cluster) postMonStartupActions() error {
}
// Create cluster-wide RBD bootstrap peer token
- _, err = controller.CreateBootstrapPeerSecret(c.context, c.ClusterInfo, &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Name: c.namespacedName.Name, Namespace: c.Namespace}}, c.ownerInfo)
- if err != nil {
+ if _, err := controller.CreateBootstrapPeerSecret(c.context, c.ClusterInfo, &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Name: c.namespacedName.Name, Namespace: c.Namespace}}, c.ownerInfo); err != nil {
return errors.Wrap(err, "failed to create cluster rbd bootstrap peer token")
}
return nil
}
+func (c *cluster) updateConfigStoreFromCRD() error {
+ monStore := config.GetMonStore(c.context, c.ClusterInfo)
+ return monStore.SetAllMultiple(c.Spec.CephConfig)
+}
+
func (c *cluster) reportTelemetry() {
// In the corner case that reconciles are started in quick succession and the telemetry
// hasn't had a chance to complete yet from a previous reconcile, simply allow
diff --git a/pkg/operator/ceph/config/monstore.go b/pkg/operator/ceph/config/monstore.go
index 96dd1f7f7..63c475e13 100644
--- a/pkg/operator/ceph/config/monstore.go
+++ b/pkg/operator/ceph/config/monstore.go
@@ -26,9 +26,13 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/util/exec"
+ "golang.org/x/exp/slices"
"gopkg.in/ini.v1"
)
+// Alias for cluster CRD ceph config options map
+type CephConfigOptionsMap = map[string]map[string]string
+
// MonStore provides methods for setting Ceph configurations in the centralized mon
// configuration database.
type MonStore struct {
@@ -205,8 +209,18 @@ func (m *MonStore) SetKeyValue(key, value string) error {
return nil
}
-func (m *MonStore) SetAll(clientName string, settings map[string]string) error {
- keys, err := m.setAll(clientName, settings)
+func (m *MonStore) SetAllMultiple(settings map[string]map[string]string) error {
+ for who, options := range settings {
+ if err := m.SetAll(who, options); err != nil {
+ return errors.Wrapf(err, "failed to set ceph config for target: %s", who)
+ }
+ }
+
+ return nil
+}
+
+func (m *MonStore) SetAll(who string, settings map[string]string) error {
+ keys, err := m.setAll(who, settings)
if err != nil {
return errors.Wrapf(err, "failed to set all keys")
}
@@ -216,13 +230,13 @@ func (m *MonStore) SetAll(clientName string, settings map[string]string) error {
logger.Infof("failed to set keys %v, trying to remove them first", keys)
newSettings := map[string]string{}
for _, key := range keys {
- if err := m.Delete(clientName, key); err != nil {
+ if err := m.Delete(who, key); err != nil {
return errors.Wrapf(err, "failed to remove key %q", key)
}
newSettings[key] = settings[key]
}
// retry setting the removed keys
- keys, err = m.setAll(clientName, newSettings)
+ keys, err = m.setAll(who, newSettings)
if err != nil {
return errors.Wrapf(err, "failed to set keys")
}
@@ -233,10 +247,10 @@ func (m *MonStore) SetAll(clientName string, settings map[string]string) error {
return nil
}
-func (m *MonStore) setAll(clientName string, settings map[string]string) ([]string, error) {
+func (m *MonStore) setAll(who string, settings map[string]string) ([]string, error) {
assimilateConfPath, err := os.CreateTemp(m.context.ConfigDir, "")
if err != nil {
- return []string{}, errors.Wrapf(err, "failed to create assimilateConf temp dir for %s.", clientName)
+ return []string{}, errors.Wrapf(err, "failed to create assimilateConf temp dir for %s.", who)
}
err = os.WriteFile(assimilateConfPath.Name(), []byte(""), 0600)
@@ -257,7 +271,7 @@ func (m *MonStore) setAll(clientName string, settings map[string]string) ([]stri
}()
configFile := ini.Empty()
- s, err := configFile.NewSection(clientName)
+ s, err := configFile.NewSection(who)
if err != nil {
return []string{}, err
}
@@ -301,12 +315,42 @@ func (m *MonStore) setAll(clientName string, settings map[string]string) ([]stri
return []string{}, errors.Wrapf(err, "failed to parse assimilate output file %s", outFilePath)
}
// get the section for the client
- section, err := iniContent.GetSection(clientName)
+ section, err := iniContent.GetSection(who)
if err != nil {
- return []string{}, errors.Wrapf(err, "failed to get section %s", clientName)
+ return []string{}, errors.Wrapf(err, "failed to get section %s", who)
}
return section.KeyStrings(), nil
}
logger.Info("successfully applied settings to the mon configuration database")
return []string{}, nil
}
+
+var criticalConfigOptions = []string{
+ "mon_host",
+ "fsid",
+ "keyring",
+}
+
+func (m *MonStore) UpdateConfigStoreFromMap(cfg CephConfigOptionsMap) error {
+ filtered := filterSettingsMap(cfg)
+
+ return m.SetAllMultiple(filtered)
+}
+
+// Filters out critical config options
+func filterSettingsMap(cfg CephConfigOptionsMap) CephConfigOptionsMap {
+ filtered := CephConfigOptionsMap{}
+
+ for who, options := range cfg {
+ filtered[who] = map[string]string{}
+ for k, v := range options {
+ if slices.Contains(criticalConfigOptions, k) {
+ continue
+ }
+
+ filtered[who][k] = v
+ }
+ }
+
+ return filtered
+}
diff --git a/pkg/operator/ceph/config/monstore_test.go b/pkg/operator/ceph/config/monstore_test.go
index ad061f532..01163f2cd 100644
--- a/pkg/operator/ceph/config/monstore_test.go
+++ b/pkg/operator/ceph/config/monstore_test.go
@@ -218,3 +218,46 @@ func TestMonStore_SetAll(t *testing.T) {
assert.NoError(t, e)
assert.True(t, appliedSettings)
}
+
+func TestFilterSettingsMap(t *testing.T) {
+ tests := []struct {
+ name string
+ input CephConfigOptionsMap
+ expected CephConfigOptionsMap
+ }{
+ {
+ name: "no filtered config options",
+ input: map[string]map[string]string{
+ "global": {
+ "osd_pool_default_size": "1",
+ },
+ },
+ expected: map[string]map[string]string{
+ "global": {
+ "osd_pool_default_size": "1",
+ },
+ },
+ },
+ {
+ name: "filtered config options",
+ input: map[string]map[string]string{
+ "global": {
+ "mon_host": "192.168.1.100:687",
+ "osd_pool_default_size": "1",
+ "bluefs_buffered_io": "false",
+ },
+ },
+ expected: map[string]map[string]string{
+ "global": {
+ "osd_pool_default_size": "1",
+ "bluefs_buffered_io": "false",
+ },
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equal(t, tt.expected, filterSettingsMap(tt.input))
+ })
+ }
+}
diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh
index 664d47900..388cd24bb 100755
--- a/tests/scripts/github-action-helper.sh
+++ b/tests/scripts/github-action-helper.sh
@@ -273,7 +273,7 @@ function deploy_cluster() {
exit 1
fi
# enable monitoring
- yq w -i -d1 cluster-test.yaml spec.monitoring.enabled true
+ yq w -i -d0 cluster-test.yaml spec.monitoring.enabled true
kubectl apply -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.40.0/bundle.yaml
kubectl create -f monitoring/rbac.yaml
@@ -401,9 +401,9 @@ function deploy_first_rook_cluster() {
cd deploy/examples/
deploy_manifest_with_local_build operator.yaml
- yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false
- yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false
- yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"1
+ yq w -i -d0 cluster-test.yaml spec.dashboard.enabled false
+ yq w -i -d0 cluster-test.yaml spec.storage.useAllDevices false
+ yq w -i -d0 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"1
kubectl create -f cluster-test.yaml
deploy_toolbox
}
@@ -413,8 +413,8 @@ function deploy_second_rook_cluster() {
cd deploy/examples/
NAMESPACE=rook-ceph-secondary envsubst