From ab80b77f495b0d78ebae6d3dea00d5e916366af4 Mon Sep 17 00:00:00 2001 From: Alexander Trost Date: Tue, 21 Nov 2023 12:11:16 +0100 Subject: [PATCH] operator: allow setting ceph config options via ceph cluster crd This implements the "Ceph Config via Ceph Cluster CRD" design document as a `cephConfig:` structure on the CRD. This also fixes the `yq` commands used to manipulate the `cluster-test.yaml` that caused CI issues for this PR and potentially unknowingly others. Signed-off-by: Alexander Trost (cherry picked from commit 4ed35d6bd671883172f3f4bfd3ec5721ff6f3f3d) --- .../CRDs/Cluster/ceph-cluster-crd.md | 29 +++++++++ Documentation/CRDs/specification.md | 24 +++++++ .../Advanced/ceph-configuration.md | 12 ++-- PendingReleaseNotes.md | 2 + .../charts/rook-ceph/templates/resources.yaml | 8 +++ deploy/examples/cluster-multus-test.yaml | 20 +++--- deploy/examples/cluster-test.yaml | 21 +++---- deploy/examples/crds.yaml | 8 +++ design/ceph/ceph-config-via-cluster-crd.md | 2 + go.mod | 2 +- pkg/apis/ceph.rook.io/v1/types.go | 5 ++ pkg/daemon/ceph/client/pool_test.go | 13 +--- pkg/operator/ceph/cluster/cluster.go | 22 ++++--- pkg/operator/ceph/config/monstore.go | 62 ++++++++++++++++--- pkg/operator/ceph/config/monstore_test.go | 43 +++++++++++++ tests/scripts/github-action-helper.sh | 12 ++-- 16 files changed, 219 insertions(+), 66 deletions(-) diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index 82935cd93..79f4e5dd0 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -92,6 +92,7 @@ For more details on the mons and when to choose a number other than `3`, see the * `removeOSDsIfOutAndSafeToRemove`: If `true` the operator will remove the OSDs that are down and whose data has been restored to other OSDs. In Ceph terms, the OSDs are `out` and `safe-to-destroy` when they are removed. * `cleanupPolicy`: [cleanup policy settings](#cleanup-policy) * `security`: [security page for key management configuration](../../Storage-Configuration/Advanced/key-management-system.md) +* `cephConfig`: [Set Ceph config options using the Ceph Mon config store](#ceph-config) ### Ceph container images @@ -937,3 +938,31 @@ However, all new configuration by the operator will be blocked with this cleanup Rook waits for the deletion of PVs provisioned using the cephCluster before proceeding to delete the cephCluster. To force deletion of the cephCluster without waiting for the PVs to be deleted, you can set the `allowUninstallWithVolumes` to true under `spec.CleanupPolicy`. + +## Ceph Config + +!!! attention + This feature is experimental. + +The Ceph config options are applied after the MONs are all in quorum and running. To set Ceph config options, you can add them to your `CephCluster` spec like this: + +```yaml +spec: + # [...] + cephConfig: + # Who's the target for these config options? + global: + # All values must be quoted so they are considered a string in YAML + osd_pool_default_size: "3" + mon_warn_on_pool_no_redundancy: "false" + osd_crush_update_on_start: "false" + # Make sure to quote special characters + "osd.*": + osd_max_scrubs: "10" +``` + +!!! warning + Rook performs no direct validation on these config options, so the validity of the settings is the + user's responsibility. + +The operator does not unset any removed config options, it is the user's responsibility to unset or set the default value for each removed option manually using the Ceph CLI. diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index f55b3c4b2..ddc9591eb 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -1113,6 +1113,18 @@ CSIDriverSpec

CSI Driver Options applied per cluster.

+ + +cephConfig
+ +map[string]map[string]string + + + +(Optional) +

Ceph Config options

+ + @@ -4389,6 +4401,18 @@ CSIDriverSpec

CSI Driver Options applied per cluster.

+ + +cephConfig
+ +map[string]map[string]string + + + +(Optional) +

Ceph Config options

+ +

ClusterState diff --git a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md index 1ba87a4dd..92a0db0a2 100644 --- a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md @@ -206,11 +206,13 @@ ceph osd pool set rbd pg_num 512 ## Custom `ceph.conf` Settings !!! warning - The advised method for controlling Ceph configuration is to manually use the Ceph CLI - or the Ceph dashboard because this offers the most flexibility. It is highly recommended that this - only be used when absolutely necessary and that the `config` be reset to an empty string if/when the - configurations are no longer necessary. Configurations in the config file will make the Ceph cluster - less configurable from the CLI and dashboard and may make future tuning or debugging difficult. + The advised method for controlling Ceph configuration is to use the [`cephConfig:` structure](../../CRDs/Cluster/ceph-cluster-crd.md#ceph-config) + in the `CephCluster` CRD. + + It is highly recommended that this only be used when absolutely necessary and that the `config` be + reset to an empty string if/when the configurations are no longer necessary. Configurations in the + config file will make the Ceph cluster less configurable from the CLI and dashboard and may make + future tuning or debugging difficult. Setting configs via Ceph's CLI requires that at least one mon be available for the configs to be set, and setting configs via dashboard requires at least one mgr to be available. Ceph also has diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 7ea0ea59a..00628c50f 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -7,3 +7,5 @@ `ROOK_DISABLE_ADMISSION_CONTROLLER: "true"` in operator.yaml before upgrading to rook v1.13. CRD validation is now enabled with [Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/). This requires Kubernetes version 1.25 or higher. ## Features + +- Added `cephConfig:` to CephCluster to allow setting Ceph config options in the Ceph MON config store via the CRD diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 2d9a514c6..c92de4811 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -895,6 +895,14 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + cephConfig: + additionalProperties: + additionalProperties: + type: string + type: object + description: Ceph Config options + nullable: true + type: object cephVersion: description: The version information that instructs Rook to orchestrate a particular version of Ceph. nullable: true diff --git a/deploy/examples/cluster-multus-test.yaml b/deploy/examples/cluster-multus-test.yaml index 3b11f8dd4..f0d87caef 100644 --- a/deploy/examples/cluster-multus-test.yaml +++ b/deploy/examples/cluster-multus-test.yaml @@ -7,19 +7,6 @@ # kubectl create -f crds.yaml -f common.yaml -f operator.yaml # kubectl create -f cluster-test.yaml ################################################################################################################# -kind: ConfigMap -apiVersion: v1 -metadata: - name: rook-config-override - namespace: rook-ceph # namespace:cluster -data: - config: | - [global] - osd_pool_default_size = 1 - mon_warn_on_pool_no_redundancy = false - bdev_flock_retry = 20 - bluefs_buffered_io = false ---- apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -47,3 +34,10 @@ spec: useAllNodes: true useAllDevices: true #deviceFilter: + cephConfig: + global: + osd_pool_default_size: "1" + mon_warn_on_pool_no_redundancy: "false" + bdev_flock_retry: "20" + bluefs_buffered_io: "false" + mon_data_avail_warn: "10" diff --git a/deploy/examples/cluster-test.yaml b/deploy/examples/cluster-test.yaml index 58c915451..4d25d0225 100644 --- a/deploy/examples/cluster-test.yaml +++ b/deploy/examples/cluster-test.yaml @@ -7,20 +7,6 @@ # kubectl create -f crds.yaml -f common.yaml -f operator.yaml # kubectl create -f cluster-test.yaml ################################################################################################################# -kind: ConfigMap -apiVersion: v1 -metadata: - name: rook-config-override - namespace: rook-ceph # namespace:cluster -data: - config: | - [global] - osd_pool_default_size = 1 - mon_warn_on_pool_no_redundancy = false - bdev_flock_retry = 20 - bluefs_buffered_io = false - mon_data_avail_warn = 10 ---- apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -57,6 +43,13 @@ spec: mgr: system-cluster-critical disruptionManagement: managePodBudgets: true + cephConfig: + global: + osd_pool_default_size: "1" + mon_warn_on_pool_no_redundancy: "false" + bdev_flock_retry: "20" + bluefs_buffered_io: "false" + mon_data_avail_warn: "10" --- apiVersion: ceph.rook.io/v1 kind: CephBlockPool diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 6e17fbb56..c747bb2bf 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -893,6 +893,14 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + cephConfig: + additionalProperties: + additionalProperties: + type: string + type: object + description: Ceph Config options + nullable: true + type: object cephVersion: description: The version information that instructs Rook to orchestrate a particular version of Ceph. nullable: true diff --git a/design/ceph/ceph-config-via-cluster-crd.md b/design/ceph/ceph-config-via-cluster-crd.md index d4b3a9dd8..43f70eb93 100644 --- a/design/ceph/ceph-config-via-cluster-crd.md +++ b/design/ceph/ceph-config-via-cluster-crd.md @@ -51,3 +51,5 @@ The operator will use the Ceph config store (that is accessed via `ceph config a ### Limitations The operator won't be unsetting any previously set config options or restore config options to their default value. + +The operator will ignore a few vital config options, such as `fsid`, `keyring` and `mon_host`. diff --git a/go.mod b/go.mod index a9f0bd6fc..79b66dbb7 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/sykesm/zap-logfmt v0.0.4 go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20230206171751-46f607a40771 golang.org/x/sync v0.5.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 @@ -116,7 +117,6 @@ require ( go.starlark.net v0.0.0-20230814145427-12f4cb8177e4 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.14.0 // indirect - golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sys v0.13.0 // indirect diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 2e6e3710f..ca5eaa797 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -226,6 +226,11 @@ type ClusterSpec struct { // CSI Driver Options applied per cluster. // +optional CSI CSIDriverSpec `json:"csi,omitempty"` + + // Ceph Config options + // +optional + // +nullable + CephConfig map[string]map[string]string `json:"cephConfig,omitempty"` } // CSIDriverSpec defines CSI Driver settings applied per cluster. diff --git a/pkg/daemon/ceph/client/pool_test.go b/pkg/daemon/ceph/client/pool_test.go index 7b101d14e..c3fa975c8 100644 --- a/pkg/daemon/ceph/client/pool_test.go +++ b/pkg/daemon/ceph/client/pool_test.go @@ -22,6 +22,8 @@ import ( "strconv" "testing" + "golang.org/x/exp/slices" + "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -214,7 +216,7 @@ func testCreateReplicaPool(t *testing.T, failureDomain, crushRoot, deviceClass, assert.Equal(t, failureDomain, args[6]) } if deviceClass == "" { - assert.False(t, testIsStringInSlice("hdd", args)) + assert.False(t, slices.Contains(args, "hdd")) } else { assert.Equal(t, deviceClass, args[7]) } @@ -336,15 +338,6 @@ func TestExtractPoolDetails(t *testing.T) { }) } -func testIsStringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - func TestGetPoolStatistics(t *testing.T) { p := PoolStatistics{} p.Images.Count = 1 diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index b3ea61502..e3e64668d 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -458,20 +458,17 @@ func (c *cluster) preMonStartupActions(cephVersion cephver.CephVersion) error { // Basically, it is executed between the monitors and the manager sequence func (c *cluster) postMonStartupActions() error { // Create CSI Kubernetes Secrets - err := csi.CreateCSISecrets(c.context, c.ClusterInfo) - if err != nil { + if err := csi.CreateCSISecrets(c.context, c.ClusterInfo); err != nil { return errors.Wrap(err, "failed to create csi kubernetes secrets") } // Create crash collector Kubernetes Secret - err = nodedaemon.CreateCrashCollectorSecret(c.context, c.ClusterInfo) - if err != nil { + if err := nodedaemon.CreateCrashCollectorSecret(c.context, c.ClusterInfo); err != nil { return errors.Wrap(err, "failed to create crash collector kubernetes secret") } // Create exporter Kubernetes Secret - err = nodedaemon.CreateExporterSecret(c.context, c.ClusterInfo) - if err != nil { + if err := nodedaemon.CreateExporterSecret(c.context, c.ClusterInfo); err != nil { return errors.Wrap(err, "failed to create exporter kubernetes secret") } @@ -479,6 +476,11 @@ func (c *cluster) postMonStartupActions() error { return errors.Wrap(err, "failed to configure msgr2") } + // Set config store options + if err := c.updateConfigStoreFromCRD(); err != nil { + return errors.Wrap(err, "") + } + crushRoot := client.GetCrushRootFromSpec(c.Spec) if crushRoot != "default" { // Remove the root=default and replicated_rule which are created by @@ -490,14 +492,18 @@ func (c *cluster) postMonStartupActions() error { } // Create cluster-wide RBD bootstrap peer token - _, err = controller.CreateBootstrapPeerSecret(c.context, c.ClusterInfo, &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Name: c.namespacedName.Name, Namespace: c.Namespace}}, c.ownerInfo) - if err != nil { + if _, err := controller.CreateBootstrapPeerSecret(c.context, c.ClusterInfo, &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Name: c.namespacedName.Name, Namespace: c.Namespace}}, c.ownerInfo); err != nil { return errors.Wrap(err, "failed to create cluster rbd bootstrap peer token") } return nil } +func (c *cluster) updateConfigStoreFromCRD() error { + monStore := config.GetMonStore(c.context, c.ClusterInfo) + return monStore.SetAllMultiple(c.Spec.CephConfig) +} + func (c *cluster) reportTelemetry() { // In the corner case that reconciles are started in quick succession and the telemetry // hasn't had a chance to complete yet from a previous reconcile, simply allow diff --git a/pkg/operator/ceph/config/monstore.go b/pkg/operator/ceph/config/monstore.go index 96dd1f7f7..63c475e13 100644 --- a/pkg/operator/ceph/config/monstore.go +++ b/pkg/operator/ceph/config/monstore.go @@ -26,9 +26,13 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/util/exec" + "golang.org/x/exp/slices" "gopkg.in/ini.v1" ) +// Alias for cluster CRD ceph config options map +type CephConfigOptionsMap = map[string]map[string]string + // MonStore provides methods for setting Ceph configurations in the centralized mon // configuration database. type MonStore struct { @@ -205,8 +209,18 @@ func (m *MonStore) SetKeyValue(key, value string) error { return nil } -func (m *MonStore) SetAll(clientName string, settings map[string]string) error { - keys, err := m.setAll(clientName, settings) +func (m *MonStore) SetAllMultiple(settings map[string]map[string]string) error { + for who, options := range settings { + if err := m.SetAll(who, options); err != nil { + return errors.Wrapf(err, "failed to set ceph config for target: %s", who) + } + } + + return nil +} + +func (m *MonStore) SetAll(who string, settings map[string]string) error { + keys, err := m.setAll(who, settings) if err != nil { return errors.Wrapf(err, "failed to set all keys") } @@ -216,13 +230,13 @@ func (m *MonStore) SetAll(clientName string, settings map[string]string) error { logger.Infof("failed to set keys %v, trying to remove them first", keys) newSettings := map[string]string{} for _, key := range keys { - if err := m.Delete(clientName, key); err != nil { + if err := m.Delete(who, key); err != nil { return errors.Wrapf(err, "failed to remove key %q", key) } newSettings[key] = settings[key] } // retry setting the removed keys - keys, err = m.setAll(clientName, newSettings) + keys, err = m.setAll(who, newSettings) if err != nil { return errors.Wrapf(err, "failed to set keys") } @@ -233,10 +247,10 @@ func (m *MonStore) SetAll(clientName string, settings map[string]string) error { return nil } -func (m *MonStore) setAll(clientName string, settings map[string]string) ([]string, error) { +func (m *MonStore) setAll(who string, settings map[string]string) ([]string, error) { assimilateConfPath, err := os.CreateTemp(m.context.ConfigDir, "") if err != nil { - return []string{}, errors.Wrapf(err, "failed to create assimilateConf temp dir for %s.", clientName) + return []string{}, errors.Wrapf(err, "failed to create assimilateConf temp dir for %s.", who) } err = os.WriteFile(assimilateConfPath.Name(), []byte(""), 0600) @@ -257,7 +271,7 @@ func (m *MonStore) setAll(clientName string, settings map[string]string) ([]stri }() configFile := ini.Empty() - s, err := configFile.NewSection(clientName) + s, err := configFile.NewSection(who) if err != nil { return []string{}, err } @@ -301,12 +315,42 @@ func (m *MonStore) setAll(clientName string, settings map[string]string) ([]stri return []string{}, errors.Wrapf(err, "failed to parse assimilate output file %s", outFilePath) } // get the section for the client - section, err := iniContent.GetSection(clientName) + section, err := iniContent.GetSection(who) if err != nil { - return []string{}, errors.Wrapf(err, "failed to get section %s", clientName) + return []string{}, errors.Wrapf(err, "failed to get section %s", who) } return section.KeyStrings(), nil } logger.Info("successfully applied settings to the mon configuration database") return []string{}, nil } + +var criticalConfigOptions = []string{ + "mon_host", + "fsid", + "keyring", +} + +func (m *MonStore) UpdateConfigStoreFromMap(cfg CephConfigOptionsMap) error { + filtered := filterSettingsMap(cfg) + + return m.SetAllMultiple(filtered) +} + +// Filters out critical config options +func filterSettingsMap(cfg CephConfigOptionsMap) CephConfigOptionsMap { + filtered := CephConfigOptionsMap{} + + for who, options := range cfg { + filtered[who] = map[string]string{} + for k, v := range options { + if slices.Contains(criticalConfigOptions, k) { + continue + } + + filtered[who][k] = v + } + } + + return filtered +} diff --git a/pkg/operator/ceph/config/monstore_test.go b/pkg/operator/ceph/config/monstore_test.go index ad061f532..01163f2cd 100644 --- a/pkg/operator/ceph/config/monstore_test.go +++ b/pkg/operator/ceph/config/monstore_test.go @@ -218,3 +218,46 @@ func TestMonStore_SetAll(t *testing.T) { assert.NoError(t, e) assert.True(t, appliedSettings) } + +func TestFilterSettingsMap(t *testing.T) { + tests := []struct { + name string + input CephConfigOptionsMap + expected CephConfigOptionsMap + }{ + { + name: "no filtered config options", + input: map[string]map[string]string{ + "global": { + "osd_pool_default_size": "1", + }, + }, + expected: map[string]map[string]string{ + "global": { + "osd_pool_default_size": "1", + }, + }, + }, + { + name: "filtered config options", + input: map[string]map[string]string{ + "global": { + "mon_host": "192.168.1.100:687", + "osd_pool_default_size": "1", + "bluefs_buffered_io": "false", + }, + }, + expected: map[string]map[string]string{ + "global": { + "osd_pool_default_size": "1", + "bluefs_buffered_io": "false", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, filterSettingsMap(tt.input)) + }) + } +} diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 664d47900..388cd24bb 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -273,7 +273,7 @@ function deploy_cluster() { exit 1 fi # enable monitoring - yq w -i -d1 cluster-test.yaml spec.monitoring.enabled true + yq w -i -d0 cluster-test.yaml spec.monitoring.enabled true kubectl apply -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.40.0/bundle.yaml kubectl create -f monitoring/rbac.yaml @@ -401,9 +401,9 @@ function deploy_first_rook_cluster() { cd deploy/examples/ deploy_manifest_with_local_build operator.yaml - yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false - yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false - yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"1 + yq w -i -d0 cluster-test.yaml spec.dashboard.enabled false + yq w -i -d0 cluster-test.yaml spec.storage.useAllDevices false + yq w -i -d0 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"1 kubectl create -f cluster-test.yaml deploy_toolbox } @@ -413,8 +413,8 @@ function deploy_second_rook_cluster() { cd deploy/examples/ NAMESPACE=rook-ceph-secondary envsubst