diff --git a/Documentation/ceph-advanced-configuration.md b/Documentation/ceph-advanced-configuration.md index 0f13cde61..27e2351b7 100644 --- a/Documentation/ceph-advanced-configuration.md +++ b/Documentation/ceph-advanced-configuration.md @@ -17,6 +17,7 @@ storage cluster. * [Separate Storage Groups](#separate-storage-groups) * [Configuring Pools](#configuring-pools) * [Custom ceph.conf Settings](#custom-cephconf-settings) +* [Custom CSI ceph.conf Settings](#custom-csi-cephconf-settings) * [OSD CRUSH Settings](#osd-crush-settings) * [OSD Dedicated Network](#osd-dedicated-network) * [Phantom OSD Removal](#phantom-osd-removal) @@ -293,6 +294,36 @@ data: osd pool default size = 2 ``` +## Custom CSI ceph.conf Settings + +> **WARNING**: It is highly recommended to use the default setting that comes with +> CephCSI and this can only be used when absolutely necessary. +> The `ceph.conf` should be reset back to default values if/when the configurations are no +> longer necessary. + +If the `csi-ceph-conf-override` ConfigMap is created before the cluster is +started, the CephCSI pods will automatically pick up the settings. If you +add the settings to the ConfigMap after the cluster has been initialized, +you can restart the Rook operator pod and wait for Rook to recreate CSI pods +to take immediate effect. + +After the CSI pods are restarted, the new settings should be in effect. + +### Example + +In this [Example](https://github.com/rook/rook/tree/{{ branchName }}/deploy/csi-ceph-conf-override.yaml) we +will set the `rbd_validate_pool` to `false` to skip rbd pool validation. + +> **WARNING**: Modify Ceph settings carefully to avoid modifying the default +> configuration. +> Changing the settings could result in unexpected results if used incorrectly. + +```console +kubectl create -f csi-ceph-conf-override.yaml +``` + +Restart the Rook operator pod and wait for CSI pods to be recreated. + ## OSD CRUSH Settings A useful view of the [CRUSH Map](http://docs.ceph.com/docs/master/rados/operations/crush-map/) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 0ab115dd1..cdab9df57 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -6,3 +6,6 @@ Pr: https://github.com/rook/rook/pull/9550 ## Features + +* Add support for custom ceph.conf for csi pods. +Pr: https://github.com/rook/rook/pull/9567 diff --git a/deploy/examples/csi-ceph-conf-override.yaml b/deploy/examples/csi-ceph-conf-override.yaml new file mode 100644 index 000000000..03139e601 --- /dev/null +++ b/deploy/examples/csi-ceph-conf-override.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: csi-ceph-conf-override + namespace: rook-ceph # namespace:operator +data: + ceph.conf: | + [global] + auth_cluster_required = cephx + auth_service_required = cephx + auth_client_required = cephx + rbd_validate_pool = false diff --git a/pkg/operator/ceph/csi/ceph_config.go b/pkg/operator/ceph/csi/ceph_config.go new file mode 100644 index 000000000..8d85d2bab --- /dev/null +++ b/pkg/operator/ceph/csi/ceph_config.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + configName = "csi-ceph-conf-override" +) + +// checkCsiCephConfigMapExists checks if the csi-ceph-conf-override configmap +// exists in the given namespace. +func checkCsiCephConfigMapExists(ctx context.Context, clientset kubernetes.Interface, namespace string) (bool, error) { + _, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{}) + if err != nil { + if !k8serrors.IsNotFound(err) { + return false, errors.Wrapf(err, "failed to get csi ceph.conf configmap %q (in %q)", configName, namespace) + } + return false, nil + } + + logger.Infof("csi ceph.conf configmap %q exists", configName) + return true, nil +} diff --git a/pkg/operator/ceph/csi/ceph_config_test.go b/pkg/operator/ceph/csi/ceph_config_test.go new file mode 100644 index 000000000..bffc4f893 --- /dev/null +++ b/pkg/operator/ceph/csi/ceph_config_test.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "testing" + + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/test" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Test_checkCsiCephConfigMapExists(t *testing.T) { + namespace := "test-namespace" + ctx := context.TODO() + // create mocked client + clientset := test.New(t, 3) + + ok, err := checkCsiCephConfigMapExists(ctx, clientset, namespace) + assert.NoError(t, err) + assert.False(t, ok) + + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName, + Namespace: namespace, + }, + } + configMap.Data = map[string]string{ + client.DefaultConfigFile: "", + } + _, err = clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}) + assert.NoError(t, err) + + ok, err = checkCsiCephConfigMapExists(ctx, clientset, namespace) + assert.NoError(t, err) + assert.True(t, ok) +} diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index ba0bd36a6..3d2c15174 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -194,6 +194,12 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map") } + exists, err := checkCsiCephConfigMapExists(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get csi ceph.conf configmap") + } + CustomCSICephConfigExists = exists + err = r.validateAndConfigureDrivers(serverVersion, ownerInfo) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi") diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index b89987e48..93b2bdd21 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -59,6 +59,7 @@ type Param struct { EnableCephFSSnapshotter bool EnableVolumeReplicationSideCar bool EnableCSIAddonsSideCar bool + MountCustomCephConf bool LogLevel uint8 CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 @@ -79,10 +80,11 @@ type templateParam struct { var ( CSIParam Param - EnableRBD = false - EnableCephFS = false - EnableCSIGRPCMetrics = false - AllowUnsupported = false + EnableRBD = false + EnableCephFS = false + EnableCSIGRPCMetrics = false + AllowUnsupported = false + CustomCSICephConfigExists = false //driver names CephFSDriverName string @@ -347,6 +349,9 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI } } + if CustomCSICephConfigExists { + tp.MountCustomCephConf = v.SupportsCustomCephConf() + } tp.ProvisionerReplicas = defaultProvisionerReplicas nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{}) if err == nil { diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index c8d7c2a37..e12918461 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -130,6 +130,11 @@ spec: mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + {{ if .MountCustomCephConf }} + - name: ceph-config + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + {{ end }} - name: liveness-prometheus image: {{ .CSIPluginImage }} args: @@ -174,3 +179,11 @@ spec: emptyDir: { medium: "Memory" } + {{ if .MountCustomCephConf }} + - name: ceph-config + configMap: + name: csi-ceph-conf-override + items: + - key: ceph.conf + path: ceph.conf + {{ end }} diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index bf9eb76db..b446da630 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -110,6 +110,11 @@ spec: mountPath: /etc/selinux readOnly: true {{ end }} + {{ if .MountCustomCephConf }} + - name: ceph-config + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + {{ end }} - name: liveness-prometheus securityContext: privileged: true @@ -176,3 +181,11 @@ spec: hostPath: path: /etc/selinux {{ end }} + {{ if .MountCustomCephConf }} + - name: ceph-config + configMap: + name: csi-ceph-conf-override + items: + - key: ceph.conf + path: ceph.conf + {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 9088d158d..d4414c9b5 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -220,6 +220,11 @@ spec: mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + {{ if .MountCustomCephConf }} + - name: ceph-config + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + {{ end }} - name: liveness-prometheus image: {{ .CSIPluginImage }} args: @@ -269,6 +274,14 @@ spec: items: - key: csi-mapping-config-json path: cluster-mapping.json + {{ if .MountCustomCephConf }} + - name: ceph-config + configMap: + name: csi-ceph-conf-override + items: + - key: ceph.conf + path: ceph.conf + {{ end }} - name: keys-tmp-dir emptyDir: { medium: "Memory" diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index e688e6af2..ea616537b 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -118,6 +118,11 @@ spec: mountPath: /etc/selinux readOnly: true {{ end }} + {{ if .MountCustomCephConf }} + - name: ceph-config + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + {{ end }} {{ if .EnableCSIAddonsSideCar }} - name: csi-addons securityContext: @@ -238,3 +243,11 @@ spec: hostPath: path: /etc/selinux {{ end }} + {{ if .MountCustomCephConf }} + - name: ceph-config + configMap: + name: csi-ceph-conf-override + items: + - key: ceph.conf + path: ceph.conf + {{ end }} diff --git a/pkg/operator/ceph/csi/version.go b/pkg/operator/ceph/csi/version.go index c8e059610..a5c277c9e 100644 --- a/pkg/operator/ceph/csi/version.go +++ b/pkg/operator/ceph/csi/version.go @@ -35,6 +35,10 @@ var ( releasev340, releasev350, } + + // custom ceph.conf is supported in v3.5.0+ + cephConfSupportedVersion = CephCSIVersion{3, 5, 0} + // for parsing the output of `cephcsi` versionCSIPattern = regexp.MustCompile(`v(\d+)\.(\d+)\.(\d+)`) ) @@ -112,3 +116,13 @@ func extractCephCSIVersion(src string) (*CephCSIVersion, error) { return &CephCSIVersion{major, minor, bugfix}, nil } + +// SupportsCustomCephConf checks if the detected version supports custom ceph.conf +func (v *CephCSIVersion) SupportsCustomCephConf() bool { + // if AllowUnsupported is set also a csi-image greater than the supported ones are allowed + if AllowUnsupported { + return true + } + + return v.isAtLeast(&cephConfSupportedVersion) +} diff --git a/pkg/operator/ceph/csi/version_test.go b/pkg/operator/ceph/csi/version_test.go index 2cca86b9f..eb987420a 100644 --- a/pkg/operator/ceph/csi/version_test.go +++ b/pkg/operator/ceph/csi/version_test.go @@ -89,3 +89,19 @@ func Test_extractCephCSIVersion(t *testing.T) { assert.Nil(t, version) assert.Contains(t, err.Error(), "failed to parse version from") } + +func TestSupportsCustomCephConf(t *testing.T) { + AllowUnsupported = true + ret := testMinVersion.SupportsCustomCephConf() + assert.True(t, ret) + + AllowUnsupported = false + ret = testMinVersion.SupportsCustomCephConf() + assert.False(t, ret) + + ret = testReleaseV340.SupportsCustomCephConf() + assert.False(t, ret) + + ret = testReleaseV350.SupportsCustomCephConf() + assert.True(t, ret) +} diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 280740c36..6de61970e 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -258,6 +258,21 @@ func (h *CephInstaller) CreateCephCluster() error { return errors.Errorf("failed to create custom ceph.conf. %+v", err) } + customCSISettings := map[string]string{ + "ceph.conf": "[global]\nauth_client_required = cephx", + } + customCSICM := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csi-ceph-conf-override", + Namespace: h.settings.Namespace, + }, + Data: customCSISettings, + } + + if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Create(ctx, customCSICM, metav1.CreateOptions{}); err != nil { + return errors.Errorf("failed to create custom csi ceph.conf. %+v", err) + } + logger.Info("Starting Rook Cluster") rookCluster := h.Manifests.GetCephCluster() logger.Info(rookCluster)