diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 4beb6a9f0..8f7b2faa6 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -128,6 +128,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port. | `9090` | | `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port. | `8080` | | `csi.enableSnapshotter` | Enable deployment of snapshotter container in ceph-csi provisioner. | `true` | +| `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs. | `true` | | `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` | | `csi.cephcsi.image` | Ceph CSI image. | `quay.io/cephcsi/cephcsi:v1.2.2` | | `csi.registrar.image` | Kubernetes CSI registrar image. | `quay.io/k8scsi/csi-node-driver-registrar:v1.1.0` | diff --git a/Documentation/k8s-pre-reqs.md b/Documentation/k8s-pre-reqs.md index b560c16de..342273e7a 100644 --- a/Documentation/k8s-pre-reqs.md +++ b/Documentation/k8s-pre-reqs.md @@ -21,9 +21,20 @@ setting up Rook in a Kubernetes cluster with Pod Security Policies enabled. The Rook agent requires setup as a Flex volume plugin to manage the storage attachments in your cluster. See the [Flex Volume Configuration](flexvolume.md) topic to configure your Kubernetes deployment to load the Rook volume plugin. -## Kernel with RBD module +## Kernel -Rook Ceph requires a Linux kernel built with the RBD module. Many distributions of Linux have this module but some don't, e.g. the GKE Container-Optimised OS (COS) does not have RBD. You can test your Kubernetes nodes by running `modprobe rbd`. If it says 'not found', you may have to [rebuild your kernel](https://rook.io/docs/rook/master/common-issues.html#rook-agent-rbd-module-missing-error) or choose a different Linux distribution. +### RBD + +Rook Ceph requires a Linux kernel built with the RBD module. Many distributions of Linux have this module but some don't, +e.g. the GKE Container-Optimised OS (COS) does not have RBD. You can test your Kubernetes nodes by running `modprobe rbd`. +If it says 'not found', you may have to [rebuild your kernel](https://rook.io/docs/rook/master/common-issues.html#rook-agent-rbd-module-missing-error) +or choose a different Linux distribution. + +### CephFS + +If you will be creating volumes from a Ceph shared file system (CephFS), the recommended minimum kernel version is 4.17. +If you have a kernel version less than 4.17, the requested PVC sizes will not be enforced. Storage quotas will only be +enforced on newer kernels. ## Kernel modules directory configuration diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index b970f5dc7..17d54b732 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -176,6 +176,10 @@ spec: - name: CSI_RBD_LIVENESS_METRICS_PORT value: {{ .Values.csi.rbdLivenessMetricsPort | quote }} {{- end }} +{{- if .Values.csi.forceCephFSKernelClient }} + - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + value: {{ .Values.csi.forceCephFSKernelClient | quote }} +{{- end }} {{- end }} - name: ROOK_ENABLE_FLEX_DRIVER value: "{{ .Values.enableFlexDriver }}" diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 83a19199a..a58c904da 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -72,6 +72,10 @@ csi: #cephfsGrpcMetricsPort: 9091 #cephfsLivenessMetricsPort: 9081 #rbdGrpcMetricsPort: 9090 + # Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS + # you may want to disable this setting. However, this will cause an issue during upgrades + # with the FUSE client. See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html + forceCephFSKernelClient: true #rbdLivenessMetricsPort: 9080 #kubeletDirPath: /var/lib/kubelet #cephcsi: @@ -138,8 +142,8 @@ hostpathRequiresPrivileged: false # Disable automatic orchestration when new devices are discovered. disableDeviceHotplug: false -# Blacklist certain disks according to the regex provided. -discoverDaemonUdev: +# Blacklist certain disks according to the regex provided. +discoverDaemonUdev: # imagePullSecrets option allow to pull docker images from private docker registry. Option will be passed to all service accounts. # imagePullSecrets: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index 27988be9e..b343a392a 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -65,6 +65,7 @@ spec: - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" - "--metricsport={{ .CephFSGRPCMetricsPort }}" + - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml index 374835e7d..42c0a776e 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml @@ -60,6 +60,7 @@ spec: - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" - "--metricsport={{ .CephFSGRPCMetricsPort }}" + - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 114ed0d58..476f69d94 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -57,6 +57,7 @@ spec: - "--mountcachedir=/mount-cache-dir" - "--pidlimit=-1" - "--metricsport={{ .CephFSGRPCMetricsPort }}" + - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 36f6f9052..64e051e25 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -235,6 +235,11 @@ spec: # Enable deployment of snapshotter container in ceph-csi provisioner - name: CSI_ENABLE_SNAPSHOTTER value: "true" + # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs + # If you disable the kernel client, your application may be disrupted during upgrade. + # See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html + - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + value: "true" # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. #- name: ROOK_CSI_KUBELET_DIR_PATH # value: "/var/lib/kubelet" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index e710b7bb4..3ee2d577e 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -185,6 +185,11 @@ spec: # Enable deployment of snapshotter container in ceph-csi provisioner. - name: CSI_ENABLE_SNAPSHOTTER value: "true" + # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs + # If you disable the kernel client, your application may be disrupted during upgrade. + # See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html + - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT + value: "true" # The default version of CSI supported by Rook will be started. To change the version # of the CSI driver to something other than what is officially supported, change # these images to the desired release of the CSI driver. @@ -252,7 +257,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - # Uncomment it to run rook operator on the host network + # Uncomment it to run rook operator on the host network #hostNetwork: true volumes: - name: rook-config diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 29d569b32..3c2e2f942 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -44,6 +44,7 @@ type Param struct { EnableSnapshotter string EnableCSIGRPCMetrics string KubeletDirPath string + ForceCephFSKernelClient string CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 RBDGRPCMetricsPort uint16 @@ -203,6 +204,13 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) + // If not set or set to anything but "false", the kernel client will be enabled + kClinet := os.Getenv("CSI_FORCE_CEPHFS_KERNEL_CLIENT") + if strings.EqualFold(kClinet, "false") { + tp.ForceCephFSKernelClient = "false" + } else { + tp.ForceCephFSKernelClient = "true" + } // parse GRPC and Liveness ports tp.CephFSGRPCMetricsPort = getPortFromENV("CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort) tp.CephFSLivenessMetricsPort = getPortFromENV("CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort)