From 7f359ba41bca9f07085bb6d495d1d07ec0b6bfeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 28 Aug 2019 23:13:56 +0200 Subject: [PATCH] operator: ceph disable flexdriver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since Ceph CSI is stable and deployed by default, we don't need to deploy the rook-ceph-agent daemonset. Signed-off-by: Sébastien Han --- Documentation/ceph-block.md | 4 ++++ Documentation/ceph-common-issues.md | 10 ++++++++++ PendingReleaseNotes.md | 2 +- cluster/charts/rook-ceph/values.yaml | 2 +- cluster/examples/kubernetes/ceph/operator.yaml | 15 +++++++++++++-- tests/framework/installer/ceph_manifests.go | 4 +++- tests/integration/ceph_base_deploy_test.go | 2 -- 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Documentation/ceph-block.md b/Documentation/ceph-block.md index 6469786bc..75f384f8a 100644 --- a/Documentation/ceph-block.md +++ b/Documentation/ceph-block.md @@ -142,6 +142,8 @@ kubectl delete storageclass rook-ceph-block ## Flex Driver To create a volume based on the flex driver instead of the CSI driver, see the following example of a storage class. +Make sure the flex driver is enabled over Ceph CSI. +For this, you need to set `ROOK_ENABLE_FLEX_DRIVER` to `true` in your operator deployment in the `operator.yaml` file. The pool definition is the same as for the CSI driver. ```yaml @@ -182,6 +184,8 @@ Continue with the example above for the [wordpress application](#consume-the-sto ## Advanced Example: Erasure Coded Block Storage +**IMPORTANT:** This is only possible when using the Flex driver. Ceph CSI 1.2 (with Rook 1.1) does not support this type of configuration yet. + If you want to use erasure coded pool with RBD, your OSDs must use `bluestore` as their `storeType`. Additionally the nodes that are going to mount the erasure coded RBD block storage must have Linux kernel >= `4.11`. diff --git a/Documentation/ceph-common-issues.md b/Documentation/ceph-common-issues.md index dbee37c10..ad175188c 100644 --- a/Documentation/ceph-common-issues.md +++ b/Documentation/ceph-common-issues.md @@ -23,6 +23,7 @@ If after trying the suggestions found on this page and the problem is not resolv - [Rook Agent rbd module missing error](#rook-agent-rbd-module-missing-error) - [Using multiple shared filesystem (CephFS) is attempted on a kernel version older than 4.7](#using-multiple-shared-filesystem-cephfs-is-attempted-on-a-kernel-version-older-than-47) - [Activate log to file for a particular Ceph daemon](#activate-log-to-file-for-a-particular-ceph-daemon) +- [Flex storage class versus Ceph CSI storage class](#flex-storage-class-versus-ceph-csi-storage-class) # Troubleshooting Techniques There are two main categories of information you will need to investigate issues in the cluster: @@ -589,3 +590,12 @@ Documentation)[Documentation/advanced-configuration.md#kubernetes] for informati the config override ConfigMap. For Ceph Luminous/Mimic releases, `mon_cluster_log_file` and `cluster_log_file` can be set to `/var/log/ceph/XXXX` in the config override ConfigMap to enable logging. See the (Advanced Documentation)[advanced-configuration.md#custom-cephconf-settings] for information about how to use the config override ConfigMap. + +# Flex storage class versus Ceph CSI storage class + +Since Rook 1.1, Ceph CSI has become stable and moving forward is the ultimate replacement over the Flex driver. +However, not all Flex storage classes are available through Ceph CSI since it's basically catching up on features. +Ceph CSI in its 1.2 version (with Rook 1.1) does not support the Erasure coded pools storage class. + +So, if you are looking at using such storage class you should enable the Flex driver by setting `ROOK_ENABLE_FLEX_DRIVER: true` in your `operator.yaml`. +Also, if you are in the need of specific features and wonder if CSI is capable of handling them, you should read [the ceph-csi support matrix](https://github.com/ceph/ceph-csi#support-matrix). diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 5bf30a8d5..ad0d2cdaf 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -20,7 +20,7 @@ an example usage ### Ceph - The minimum version supported by Rook is now Ceph Mimic v13.2.4. -- The Ceph CSI driver is enabled by default and preferred over the flex driver +- The Ceph CSI driver is enabled by default and preferred over the flex driver which has been disabled by default - The flex driver can be disabled in operator.yaml by setting ROOK_ENABLE_FLEX_DRIVER=false - The CSI drivers can be disabled by setting ROOK_CSI_ENABLE_CEPHFS=false and ROOK_CSI_ENABLE_RBD=false - The device discovery daemon can be disabled in operator.yaml by setting ROOK_ENABLE_DISCOVERY_DAEMON=false diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 5d4edae10..e7d0f3c96 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -57,7 +57,7 @@ pspEnable: true enableCsiRbdDriver: true enableCsiCephfsDriver: true enableCsiGrpcMetrics: true -enableFlexDriver: true +enableFlexDriver: false enableDiscoveryDaemon: true ## Rook Agent configuration diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index d9a493763..63a582781 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -106,36 +106,45 @@ spec: # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027 - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS value: "false" + # The logging level for the operator: INFO | DEBUG - name: ROOK_LOG_LEVEL value: "INFO" + # The interval to check the health of the ceph cluster and update the status in the custom resource. - name: ROOK_CEPH_STATUS_CHECK_INTERVAL value: "60s" + # The interval to check if every mon is in the quorum. - name: ROOK_MON_HEALTHCHECK_INTERVAL value: "45s" + # The duration to wait before trying to failover or remove/replace the # current mon with a new mon (useful for compensating flapping network). - name: ROOK_MON_OUT_TIMEOUT value: "600s" + # The duration between discovering devices in the rook-discover daemonset. - name: ROOK_DISCOVER_DEVICES_INTERVAL value: "60m" + # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods. # This is necessary to workaround the anyuid issues when running on OpenShift. # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED value: "false" + # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins). # Disable it here if you have similar issues. # For more details see https://github.com/rook/rook/issues/2417 - name: ROOK_ENABLE_SELINUX_RELABELING value: "true" + # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues. # For more details see https://github.com/rook/rook/issues/2254 - name: ROOK_ENABLE_FSGROUP value: "true" + # Disable automatic orchestration when new devices are discovered - name: ROOK_DISABLE_DEVICE_HOTPLUG value: "false" @@ -143,16 +152,18 @@ spec: # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release # in favor of the CSI driver. - name: ROOK_ENABLE_FLEX_DRIVER - value: "true" + value: "false" # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. - name: ROOK_ENABLE_DISCOVERY_DAEMON value: "true" - # Enable the default version of the CSI driver. To start another version of the CSI driver, see image properties below. + # Enable the default version of the CSI CephFS driver. To start another version of the CSI driver, see image properties below. - name: ROOK_CSI_ENABLE_CEPHFS value: "true" + + # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below. - name: ROOK_CSI_ENABLE_RBD value: "true" - name: ROOK_CSI_ENABLE_GRPC_METRICS diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 35af3141c..3927c1726 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -739,7 +739,7 @@ rules: - get - list --- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location +# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: @@ -1364,6 +1364,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: ROOK_ENABLE_FLEX_DRIVER + value: "true" - name: ROOK_CSI_ENABLE_CEPHFS value: "true" - name: ROOK_CSI_ENABLE_RBD diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go index 8d6645bb3..996bb9a3b 100644 --- a/tests/integration/ceph_base_deploy_test.go +++ b/tests/integration/ceph_base_deploy_test.go @@ -45,8 +45,6 @@ func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNames logger.Infof("Make sure all Pods in Rook Cluster %s are running", clusterNamespace) assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-operator", opNamespace, 1, "Running"), "Make sure there is 1 rook-operator present in Running state") - assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-agent", opNamespace, 1, "Running"), - "Make sure there is 1 rook-ceph-agent present in Running state") assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", clusterNamespace, 1, "Running"), "Make sure there is 1 rook-ceph-mgr present in Running state") assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-osd", clusterNamespace, 1, "Running"),