From 2076a3c788abefd3578d8ec18cf8668555a972f6 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 13 Dec 2021 18:30:19 +0530 Subject: [PATCH] csi: add custom ceph.conf for csi pods Currently, the admin/user can configure the ceph.conf for daemon pods using https://rook.io/docs/rook/v1.7/ ceph-advanced-configuration.html#custom-cephconf-settings. This the above custom ceph.conf is only for ceph pods. the support to provide constom ceph.conf for cephcsi is added in cephcsi PR 2476 This PR adds the support to create/update ceph.conf for csi pods. Signed-off-by: Madhu Rajanna --- Documentation/ceph-advanced-configuration.md | 31 ++++++++++ PendingReleaseNotes.md | 3 + deploy/examples/csi-ceph-conf-override.yaml | 12 ++++ pkg/operator/ceph/csi/ceph_config.go | 45 +++++++++++++++ pkg/operator/ceph/csi/ceph_config_test.go | 56 +++++++++++++++++++ pkg/operator/ceph/csi/controller.go | 6 ++ pkg/operator/ceph/csi/spec.go | 13 +++-- .../csi-cephfsplugin-provisioner-dep.yaml | 13 +++++ .../csi/template/cephfs/csi-cephfsplugin.yaml | 13 +++++ .../rbd/csi-rbdplugin-provisioner-dep.yaml | 13 +++++ .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 13 +++++ pkg/operator/ceph/csi/version.go | 14 +++++ pkg/operator/ceph/csi/version_test.go | 16 ++++++ tests/framework/installer/ceph_installer.go | 15 +++++ 14 files changed, 259 insertions(+), 4 deletions(-) create mode 100644 deploy/examples/csi-ceph-conf-override.yaml create mode 100644 pkg/operator/ceph/csi/ceph_config.go create mode 100644 pkg/operator/ceph/csi/ceph_config_test.go diff --git a/Documentation/ceph-advanced-configuration.md b/Documentation/ceph-advanced-configuration.md index 0f13cde61..27e2351b7 100644 --- a/Documentation/ceph-advanced-configuration.md +++ b/Documentation/ceph-advanced-configuration.md @@ -17,6 +17,7 @@ storage cluster. * [Separate Storage Groups](#separate-storage-groups) * [Configuring Pools](#configuring-pools) * [Custom ceph.conf Settings](#custom-cephconf-settings) +* [Custom CSI ceph.conf Settings](#custom-csi-cephconf-settings) * [OSD CRUSH Settings](#osd-crush-settings) * [OSD Dedicated Network](#osd-dedicated-network) * [Phantom OSD Removal](#phantom-osd-removal) @@ -293,6 +294,36 @@ data: osd pool default size = 2 ``` +## Custom CSI ceph.conf Settings + +> **WARNING**: It is highly recommended to use the default setting that comes with +> CephCSI and this can only be used when absolutely necessary. +> The `ceph.conf` should be reset back to default values if/when the configurations are no +> longer necessary. + +If the `csi-ceph-conf-override` ConfigMap is created before the cluster is +started, the CephCSI pods will automatically pick up the settings. If you +add the settings to the ConfigMap after the cluster has been initialized, +you can restart the Rook operator pod and wait for Rook to recreate CSI pods +to take immediate effect. + +After the CSI pods are restarted, the new settings should be in effect. + +### Example + +In this [Example](https://github.com/rook/rook/tree/{{ branchName }}/deploy/csi-ceph-conf-override.yaml) we +will set the `rbd_validate_pool` to `false` to skip rbd pool validation. + +> **WARNING**: Modify Ceph settings carefully to avoid modifying the default +> configuration. +> Changing the settings could result in unexpected results if used incorrectly. + +```console +kubectl create -f csi-ceph-conf-override.yaml +``` + +Restart the Rook operator pod and wait for CSI pods to be recreated. + ## OSD CRUSH Settings A useful view of the [CRUSH Map](http://docs.ceph.com/docs/master/rados/operations/crush-map/) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 0ab115dd1..cdab9df57 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -6,3 +6,6 @@ Pr: https://github.com/rook/rook/pull/9550 ## Features + +* Add support for custom ceph.conf for csi pods. +Pr: https://github.com/rook/rook/pull/9567 diff --git a/deploy/examples/csi-ceph-conf-override.yaml b/deploy/examples/csi-ceph-conf-override.yaml new file mode 100644 index 000000000..03139e601 --- /dev/null +++ b/deploy/examples/csi-ceph-conf-override.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: csi-ceph-conf-override + namespace: rook-ceph # namespace:operator +data: + ceph.conf: | + [global] + auth_cluster_required = cephx + auth_service_required = cephx + auth_client_required = cephx + rbd_validate_pool = false diff --git a/pkg/operator/ceph/csi/ceph_config.go b/pkg/operator/ceph/csi/ceph_config.go new file mode 100644 index 000000000..8d85d2bab --- /dev/null +++ b/pkg/operator/ceph/csi/ceph_config.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + configName = "csi-ceph-conf-override" +) + +// checkCsiCephConfigMapExists checks if the csi-ceph-conf-override configmap +// exists in the given namespace. +func checkCsiCephConfigMapExists(ctx context.Context, clientset kubernetes.Interface, namespace string) (bool, error) { + _, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{}) + if err != nil { + if !k8serrors.IsNotFound(err) { + return false, errors.Wrapf(err, "failed to get csi ceph.conf configmap %q (in %q)", configName, namespace) + } + return false, nil + } + + logger.Infof("csi ceph.conf configmap %q exists", configName) + return true, nil +} diff --git a/pkg/operator/ceph/csi/ceph_config_test.go b/pkg/operator/ceph/csi/ceph_config_test.go new file mode 100644 index 000000000..bffc4f893 --- /dev/null +++ b/pkg/operator/ceph/csi/ceph_config_test.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "testing" + + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/test" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Test_checkCsiCephConfigMapExists(t *testing.T) { + namespace := "test-namespace" + ctx := context.TODO() + // create mocked client + clientset := test.New(t, 3) + + ok, err := checkCsiCephConfigMapExists(ctx, clientset, namespace) + assert.NoError(t, err) + assert.False(t, ok) + + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName, + Namespace: namespace, + }, + } + configMap.Data = map[string]string{ + client.DefaultConfigFile: "", + } + _, err = clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}) + assert.NoError(t, err) + + ok, err = checkCsiCephConfigMapExists(ctx, clientset, namespace) + assert.NoError(t, err) + assert.True(t, ok) +} diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index ba0bd36a6..3d2c15174 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -194,6 +194,12 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map") } + exists, err := checkCsiCephConfigMapExists(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get csi ceph.conf configmap") + } + CustomCSICephConfigExists = exists + err = r.validateAndConfigureDrivers(serverVersion, ownerInfo) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi") diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index b89987e48..93b2bdd21 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -59,6 +59,7 @@ type Param struct { EnableCephFSSnapshotter bool EnableVolumeReplicationSideCar bool EnableCSIAddonsSideCar bool + MountCustomCephConf bool LogLevel uint8 CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 @@ -79,10 +80,11 @@ type templateParam struct { var ( CSIParam Param - EnableRBD = false - EnableCephFS = false - EnableCSIGRPCMetrics = false - AllowUnsupported = false + EnableRBD = false + EnableCephFS = false + EnableCSIGRPCMetrics = false + AllowUnsupported = false + CustomCSICephConfigExists = false //driver names CephFSDriverName string @@ -347,6 +349,9 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI } } + if CustomCSICephConfigExists { + tp.MountCustomCephConf = v.SupportsCustomCephConf() + } tp.ProvisionerReplicas = defaultProvisionerReplicas nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{}) if err == nil { diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index c8d7c2a37..e12918461 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -130,6 +130,11 @@ spec: mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + {{ if .MountCustomCephConf }} + - name: ceph-config + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + {{ end }} - name: liveness-prometheus image: {{ .CSIPluginImage }} args: @@ -174,3 +179,11 @@ spec: emptyDir: { medium: "Memory" } + {{ if .MountCustomCephConf }} + - name: ceph-config + configMap: + name: csi-ceph-conf-override + items: + - key: ceph.conf + path: ceph.conf + {{ end }} diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index bf9eb76db..b446da630 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -110,6 +110,11 @@ spec: mountPath: /etc/selinux readOnly: true {{ end }} + {{ if .MountCustomCephConf }} + - name: ceph-config + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + {{ end }} - name: liveness-prometheus securityContext: privileged: true @@ -176,3 +181,11 @@ spec: hostPath: path: /etc/selinux {{ end }} + {{ if .MountCustomCephConf }} + - name: ceph-config + configMap: + name: csi-ceph-conf-override + items: + - key: ceph.conf + path: ceph.conf + {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 9088d158d..d4414c9b5 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -220,6 +220,11 @@ spec: mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + {{ if .MountCustomCephConf }} + - name: ceph-config + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + {{ end }} - name: liveness-prometheus image: {{ .CSIPluginImage }} args: @@ -269,6 +274,14 @@ spec: items: - key: csi-mapping-config-json path: cluster-mapping.json + {{ if .MountCustomCephConf }} + - name: ceph-config + configMap: + name: csi-ceph-conf-override + items: + - key: ceph.conf + path: ceph.conf + {{ end }} - name: keys-tmp-dir emptyDir: { medium: "Memory" diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index e688e6af2..ea616537b 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -118,6 +118,11 @@ spec: mountPath: /etc/selinux readOnly: true {{ end }} + {{ if .MountCustomCephConf }} + - name: ceph-config + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + {{ end }} {{ if .EnableCSIAddonsSideCar }} - name: csi-addons securityContext: @@ -238,3 +243,11 @@ spec: hostPath: path: /etc/selinux {{ end }} + {{ if .MountCustomCephConf }} + - name: ceph-config + configMap: + name: csi-ceph-conf-override + items: + - key: ceph.conf + path: ceph.conf + {{ end }} diff --git a/pkg/operator/ceph/csi/version.go b/pkg/operator/ceph/csi/version.go index c8e059610..a5c277c9e 100644 --- a/pkg/operator/ceph/csi/version.go +++ b/pkg/operator/ceph/csi/version.go @@ -35,6 +35,10 @@ var ( releasev340, releasev350, } + + // custom ceph.conf is supported in v3.5.0+ + cephConfSupportedVersion = CephCSIVersion{3, 5, 0} + // for parsing the output of `cephcsi` versionCSIPattern = regexp.MustCompile(`v(\d+)\.(\d+)\.(\d+)`) ) @@ -112,3 +116,13 @@ func extractCephCSIVersion(src string) (*CephCSIVersion, error) { return &CephCSIVersion{major, minor, bugfix}, nil } + +// SupportsCustomCephConf checks if the detected version supports custom ceph.conf +func (v *CephCSIVersion) SupportsCustomCephConf() bool { + // if AllowUnsupported is set also a csi-image greater than the supported ones are allowed + if AllowUnsupported { + return true + } + + return v.isAtLeast(&cephConfSupportedVersion) +} diff --git a/pkg/operator/ceph/csi/version_test.go b/pkg/operator/ceph/csi/version_test.go index 2cca86b9f..eb987420a 100644 --- a/pkg/operator/ceph/csi/version_test.go +++ b/pkg/operator/ceph/csi/version_test.go @@ -89,3 +89,19 @@ func Test_extractCephCSIVersion(t *testing.T) { assert.Nil(t, version) assert.Contains(t, err.Error(), "failed to parse version from") } + +func TestSupportsCustomCephConf(t *testing.T) { + AllowUnsupported = true + ret := testMinVersion.SupportsCustomCephConf() + assert.True(t, ret) + + AllowUnsupported = false + ret = testMinVersion.SupportsCustomCephConf() + assert.False(t, ret) + + ret = testReleaseV340.SupportsCustomCephConf() + assert.False(t, ret) + + ret = testReleaseV350.SupportsCustomCephConf() + assert.True(t, ret) +} diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 280740c36..6de61970e 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -258,6 +258,21 @@ func (h *CephInstaller) CreateCephCluster() error { return errors.Errorf("failed to create custom ceph.conf. %+v", err) } + customCSISettings := map[string]string{ + "ceph.conf": "[global]\nauth_client_required = cephx", + } + customCSICM := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csi-ceph-conf-override", + Namespace: h.settings.Namespace, + }, + Data: customCSISettings, + } + + if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Create(ctx, customCSICM, metav1.CreateOptions{}); err != nil { + return errors.Errorf("failed to create custom csi ceph.conf. %+v", err) + } + logger.Info("Starting Rook Cluster") rookCluster := h.Manifests.GetCephCluster() logger.Info(rookCluster)