From 85d622dc2f8507a5936d93e3df2167c6fe67b551 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 29 Nov 2019 10:24:18 +0530 Subject: [PATCH 1/2] CSI: Add ENV variable to use ceph kernel client In ceph-csi v1.2.2 we have add a flag `forcecephkernelclient` to force use Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs Signed-off-by: Madhu Rajanna --- Documentation/ceph-upgrade.md | 9 +++++++++ Documentation/helm-operator.md | 1 + cluster/charts/rook-ceph/templates/deployment.yaml | 4 ++++ cluster/charts/rook-ceph/values.yaml | 2 ++ .../cephfs/csi-cephfsplugin-provisioner-dep.yaml | 1 + .../cephfs/csi-cephfsplugin-provisioner-sts.yaml | 1 + .../ceph/csi/template/cephfs/csi-cephfsplugin.yaml | 1 + cluster/examples/kubernetes/ceph/operator-openshift.yaml | 3 +++ cluster/examples/kubernetes/ceph/operator.yaml | 3 +++ pkg/operator/ceph/csi/spec.go | 7 +++++++ 10 files changed, 32 insertions(+) diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index d042c9165..1f5c72086 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -425,6 +425,15 @@ located. value: "quay.io/k8scsi/csi-attacher:v1.2.0" ``` +If you want to enable Ceph Kernel clients on kernel < 4.17 which support quotas +for Cephfs please add below to the operator `env` variables. + +```yaml + env: + - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + value: "true" +``` + ### Verifying updates You can use the below command to see the CSI images currently being used in the cluster. Once there is only a single version of each image and the version is the latest one you expect, the CSI pods diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 4beb6a9f0..8f7b2faa6 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -128,6 +128,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port. | `9090` | | `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port. | `8080` | | `csi.enableSnapshotter` | Enable deployment of snapshotter container in ceph-csi provisioner. | `true` | +| `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs. | `true` | | `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` | | `csi.cephcsi.image` | Ceph CSI image. | `quay.io/cephcsi/cephcsi:v1.2.2` | | `csi.registrar.image` | Kubernetes CSI registrar image. | `quay.io/k8scsi/csi-node-driver-registrar:v1.1.0` | diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index b970f5dc7..17d54b732 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -176,6 +176,10 @@ spec: - name: CSI_RBD_LIVENESS_METRICS_PORT value: {{ .Values.csi.rbdLivenessMetricsPort | quote }} {{- end }} +{{- if .Values.csi.forceCephFSKernelClient }} + - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + value: {{ .Values.csi.forceCephFSKernelClient | quote }} +{{- end }} {{- end }} - name: ROOK_ENABLE_FLEX_DRIVER value: "{{ .Values.enableFlexDriver }}" diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 83a19199a..2da1a3351 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -72,6 +72,8 @@ csi: #cephfsGrpcMetricsPort: 9091 #cephfsLivenessMetricsPort: 9081 #rbdGrpcMetricsPort: 9090 + # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs + #forceCephFSKernelClient: true #rbdLivenessMetricsPort: 9080 #kubeletDirPath: /var/lib/kubelet #cephcsi: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index 27988be9e..b343a392a 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -65,6 +65,7 @@ spec: - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" - "--metricsport={{ .CephFSGRPCMetricsPort }}" + - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml index 374835e7d..42c0a776e 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml @@ -60,6 +60,7 @@ spec: - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" - "--metricsport={{ .CephFSGRPCMetricsPort }}" + - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 114ed0d58..476f69d94 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -57,6 +57,7 @@ spec: - "--mountcachedir=/mount-cache-dir" - "--pidlimit=-1" - "--metricsport={{ .CephFSGRPCMetricsPort }}" + - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 36f6f9052..c55d7f22c 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -235,6 +235,9 @@ spec: # Enable deployment of snapshotter container in ceph-csi provisioner - name: CSI_ENABLE_SNAPSHOTTER value: "true" + # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs + # - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + # value: "true" # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. #- name: ROOK_CSI_KUBELET_DIR_PATH # value: "/var/lib/kubelet" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index e710b7bb4..76d01930f 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -185,6 +185,9 @@ spec: # Enable deployment of snapshotter container in ceph-csi provisioner. - name: CSI_ENABLE_SNAPSHOTTER value: "true" + # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs + # - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + # value: "true" # The default version of CSI supported by Rook will be started. To change the version # of the CSI driver to something other than what is officially supported, change # these images to the desired release of the CSI driver. diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 29d569b32..6c8df16c9 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -44,6 +44,7 @@ type Param struct { EnableSnapshotter string EnableCSIGRPCMetrics string KubeletDirPath string + ForceCephFSKernelClient string CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 RBDGRPCMetricsPort uint16 @@ -203,6 +204,12 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) + kClinet := os.Getenv("CSI_FORCE_CEPHFS_KERNEL_CLIENT") + if strings.EqualFold(kClinet, "true") { + tp.ForceCephFSKernelClient = "true" + } else { + tp.ForceCephFSKernelClient = "false" + } // parse GRPC and Liveness ports tp.CephFSGRPCMetricsPort = getPortFromENV("CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort) tp.CephFSLivenessMetricsPort = getPortFromENV("CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) From 804e41132d44f05e3424ae3f421c46cecb93e1a1 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 13 Dec 2019 16:33:04 -0700 Subject: [PATCH 2/2] ceph: force enable the cephfs kernel driver by default There are two issues at odds with the cephfs driver. The first is that older kernels less than 4.17 do not support quotas so PVC sizes will not be enforced with the kernel driver in that case. If the FUSE client is used instead, there are issues during upgrade because the client mount will be disrupted and the client pods will have to restart. To prevent client pods from breaking during upgrade, we enable the kernel driver by default. If the quotas are critical, the admin will need to take steps during upgrade to prevent breaking the applications. Signed-off-by: Travis Nielsen --- Documentation/ceph-upgrade.md | 9 --------- Documentation/k8s-pre-reqs.md | 15 +++++++++++++-- cluster/charts/rook-ceph/values.yaml | 10 ++++++---- .../kubernetes/ceph/operator-openshift.yaml | 6 ++++-- cluster/examples/kubernetes/ceph/operator.yaml | 8 +++++--- pkg/operator/ceph/csi/spec.go | 7 ++++--- 6 files changed, 32 insertions(+), 23 deletions(-) diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index 1f5c72086..d042c9165 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -425,15 +425,6 @@ located. value: "quay.io/k8scsi/csi-attacher:v1.2.0" ``` -If you want to enable Ceph Kernel clients on kernel < 4.17 which support quotas -for Cephfs please add below to the operator `env` variables. - -```yaml - env: - - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT - value: "true" -``` - ### Verifying updates You can use the below command to see the CSI images currently being used in the cluster. Once there is only a single version of each image and the version is the latest one you expect, the CSI pods diff --git a/Documentation/k8s-pre-reqs.md b/Documentation/k8s-pre-reqs.md index b560c16de..342273e7a 100644 --- a/Documentation/k8s-pre-reqs.md +++ b/Documentation/k8s-pre-reqs.md @@ -21,9 +21,20 @@ setting up Rook in a Kubernetes cluster with Pod Security Policies enabled. The Rook agent requires setup as a Flex volume plugin to manage the storage attachments in your cluster. See the [Flex Volume Configuration](flexvolume.md) topic to configure your Kubernetes deployment to load the Rook volume plugin. -## Kernel with RBD module +## Kernel -Rook Ceph requires a Linux kernel built with the RBD module. Many distributions of Linux have this module but some don't, e.g. the GKE Container-Optimised OS (COS) does not have RBD. You can test your Kubernetes nodes by running `modprobe rbd`. If it says 'not found', you may have to [rebuild your kernel](https://rook.io/docs/rook/master/common-issues.html#rook-agent-rbd-module-missing-error) or choose a different Linux distribution. +### RBD + +Rook Ceph requires a Linux kernel built with the RBD module. Many distributions of Linux have this module but some don't, +e.g. the GKE Container-Optimised OS (COS) does not have RBD. You can test your Kubernetes nodes by running `modprobe rbd`. +If it says 'not found', you may have to [rebuild your kernel](https://rook.io/docs/rook/master/common-issues.html#rook-agent-rbd-module-missing-error) +or choose a different Linux distribution. + +### CephFS + +If you will be creating volumes from a Ceph shared file system (CephFS), the recommended minimum kernel version is 4.17. +If you have a kernel version less than 4.17, the requested PVC sizes will not be enforced. Storage quotas will only be +enforced on newer kernels. ## Kernel modules directory configuration diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 2da1a3351..a58c904da 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -72,8 +72,10 @@ csi: #cephfsGrpcMetricsPort: 9091 #cephfsLivenessMetricsPort: 9081 #rbdGrpcMetricsPort: 9090 - # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs - #forceCephFSKernelClient: true + # Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS + # you may want to disable this setting. However, this will cause an issue during upgrades + # with the FUSE client. See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html + forceCephFSKernelClient: true #rbdLivenessMetricsPort: 9080 #kubeletDirPath: /var/lib/kubelet #cephcsi: @@ -140,8 +142,8 @@ hostpathRequiresPrivileged: false # Disable automatic orchestration when new devices are discovered. disableDeviceHotplug: false -# Blacklist certain disks according to the regex provided. -discoverDaemonUdev: +# Blacklist certain disks according to the regex provided. +discoverDaemonUdev: # imagePullSecrets option allow to pull docker images from private docker registry. Option will be passed to all service accounts. # imagePullSecrets: diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index c55d7f22c..64e051e25 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -236,8 +236,10 @@ spec: - name: CSI_ENABLE_SNAPSHOTTER value: "true" # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs - # - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT - # value: "true" + # If you disable the kernel client, your application may be disrupted during upgrade. + # See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html + - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + value: "true" # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. #- name: ROOK_CSI_KUBELET_DIR_PATH # value: "/var/lib/kubelet" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 76d01930f..3ee2d577e 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -186,8 +186,10 @@ spec: - name: CSI_ENABLE_SNAPSHOTTER value: "true" # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs - # - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT - # value: "true" + # If you disable the kernel client, your application may be disrupted during upgrade. + # See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html + - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + value: "true" # The default version of CSI supported by Rook will be started. To change the version # of the CSI driver to something other than what is officially supported, change # these images to the desired release of the CSI driver. @@ -255,7 +257,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - # Uncomment it to run rook operator on the host network + # Uncomment it to run rook operator on the host network #hostNetwork: true volumes: - name: rook-config diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 6c8df16c9..3c2e2f942 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -204,11 +204,12 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) + // If not set or set to anything but "false", the kernel client will be enabled kClinet := os.Getenv("CSI_FORCE_CEPHFS_KERNEL_CLIENT") - if strings.EqualFold(kClinet, "true") { - tp.ForceCephFSKernelClient = "true" - } else { + if strings.EqualFold(kClinet, "false") { tp.ForceCephFSKernelClient = "false" + } else { + tp.ForceCephFSKernelClient = "true" } // parse GRPC and Liveness ports tp.CephFSGRPCMetricsPort = getPortFromENV("CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort)