From cc6ec5889119a874681b8528ca73f7681cd164ce Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 5 May 2020 12:23:36 +0530 Subject: [PATCH 1/2] CSI: Set PriorityClassName for CSI plugin pods provide an option to set Priorityclassname on the CSI plugin pods to avoid pod eviction, as the CSI plugin pods are critical in system for storage. the default value for the CSI plugin pods is set to system-node-critical. more info can be found at https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ Signed-off-by: Madhu Rajanna --- Documentation/helm-operator.md | 1 + cluster/charts/rook-ceph/templates/deployment.yaml | 2 ++ cluster/charts/rook-ceph/values.yaml | 2 ++ .../ceph/csi/template/cephfs/csi-cephfsplugin.yaml | 3 +++ .../kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml | 3 +++ cluster/examples/kubernetes/ceph/operator-openshift.yaml | 3 +++ cluster/examples/kubernetes/ceph/operator.yaml | 3 +++ pkg/operator/ceph/csi/spec.go | 7 +++++++ 8 files changed, 24 insertions(+) diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 5dd910300..a2224db97 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -119,6 +119,7 @@ The following tables lists the configurable parameters of the rook-operator char | `discover.nodeAffinity` | The node labels for affinity of `discover-agent` (***) | | | `csi.enableRbdDriver` | Enable Ceph CSI RBD driver. | `true` | | `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver. | `true` | +| `csi.pluginPriorityClassName` | PriorityClassName to be set on csi driver plugin pods. | | | `csi.logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | | `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics. | `true` | | `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment. | | diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 9e6fcde5e..0d0f3349f 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -108,6 +108,8 @@ spec: value: {{ .Values.csi.enableCephfsDriver | quote }} - name: CSI_ENABLE_SNAPSHOTTER value: {{ .Values.csi.enableSnapshotter | quote }} + - name: CSI_PLUGIN_PRIORITY_CLASSNAME + value: {{ .Values.csi.pluginPriorityClassName | quote }} {{- if .Values.csi.cephFSPluginUpdateStrategy }} - name: CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY value: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }} diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index dcd6d29ef..60fe87330 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -58,6 +58,8 @@ csi: enableCephfsDriver: true enableGrpcMetrics: true enableSnapshotter: true + # (Optional) set user created priorityclassName for csi plugin pods. + # pluginPriorityClassName: system-node-critical # Set logging level for csi containers. # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. #logLevel: 0 diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 7217eeeac..77b8aa7dc 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -17,6 +17,9 @@ spec: spec: serviceAccount: rook-csi-cephfs-plugin-sa hostNetwork: true + {{ if .PluginPriorityClassName }} + priorityClassName: {{ .PluginPriorityClassName }} + {{ end }} # to use e.g. Rook orchestrated cluster, and mons' FQDN is # resolved through k8s service, set dns policy to cluster first dnsPolicy: ClusterFirstWithHostNet diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index be9731578..d0ea98a7c 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -16,6 +16,9 @@ spec: contains: csi-rbdplugin-metrics spec: serviceAccount: rook-csi-rbd-plugin-sa + {{ if .PluginPriorityClassName }} + priorityClassName: {{ .PluginPriorityClassName }} + {{ end }} hostNetwork: true hostPID: true # to use e.g. Rook orchestrated cluster, and mons' FQDN is diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 30e82a94a..64c50d818 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -129,6 +129,9 @@ data: # ROOK_CSI_SNAPSHOTTER_IMAGE: "quay.io/k8scsi/csi-snapshotter:v1.2.2" # ROOK_CSI_ATTACHER_IMAGE: "quay.io/k8scsi/csi-attacher:v2.1.0" + # (Optional) set user created priorityclassName for csi plugin pods. + # CSI_PLUGIN_PRIORITY_CLASSNAME="system-node-critical" + # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 81691e553..c82eaf0fe 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -53,6 +53,9 @@ data: # ROOK_CSI_SNAPSHOTTER_IMAGE: "quay.io/k8scsi/csi-snapshotter:v1.2.2" # ROOK_CSI_ATTACHER_IMAGE: "quay.io/k8scsi/csi-attacher:v2.1.0" + # (Optional) set user created priorityclassName for csi plugin pods. + # CSI_PLUGIN_PRIORITY_CLASSNAME="system-node-critical" + # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 9ec38f46f..790452b6a 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -53,6 +53,7 @@ type Param struct { ForceCephFSKernelClient string CephFSPluginUpdateStrategy string RBDPluginUpdateStrategy string + PluginPriorityClassName string LogLevel uint8 CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 @@ -273,6 +274,12 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers tp.EnableSnapshotter = "true" } + // default value `system-node-critical` is the highest available priority + tp.PluginPriorityClassName, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PLUGIN_PRIORITY_CLASSNAME", "") + if err != nil { + return errors.Wrap(err, "failed to load CSI_PLUGIN_PRIORITY_CLASSNAME setting") + } + updateStrategy, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate) if err != nil { return errors.Wrap(err, "failed to load CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY setting") From ec3ab55d94d8af78b3150d68410f5b828039cd01 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 5 May 2020 13:25:48 +0530 Subject: [PATCH 2/2] CSI: Set PriorityClassName for CSI provisioner pods provide an option to set PriorityclassName on the CSI provisioner pods to avoid pod eviction, as the CSI provisioner pods a critical in system for storage. the default value for the CSI provisioner pods is set to system-cluster-critical. provisioner pods are less priority than plugin pods. Signed-off-by: Madhu Rajanna --- Documentation/helm-operator.md | 1 + .../rook-ceph/templates/deployment.yaml | 2 + cluster/charts/rook-ceph/values.yaml | 4 ++ .../csi-cephfsplugin-provisioner-dep.yaml | 3 ++ .../csi-cephfsplugin-provisioner-sts.yaml | 3 ++ .../rbd/csi-rbdplugin-provisioner-dep.yaml | 3 ++ .../rbd/csi-rbdplugin-provisioner-sts.yaml | 3 ++ .../kubernetes/ceph/operator-openshift.yaml | 3 ++ .../examples/kubernetes/ceph/operator.yaml | 3 ++ pkg/operator/ceph/csi/spec.go | 46 +++++++++++-------- 10 files changed, 52 insertions(+), 19 deletions(-) diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index a2224db97..93524692c 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -120,6 +120,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.enableRbdDriver` | Enable Ceph CSI RBD driver. | `true` | | `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver. | `true` | | `csi.pluginPriorityClassName` | PriorityClassName to be set on csi driver plugin pods. | | +| `csi.provisionerPriorityClassName` | PriorityClassName to be set on csi driver provisioner pods. | | | `csi.logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | | `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics. | `true` | | `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment. | | diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 0d0f3349f..ae2055a30 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -110,6 +110,8 @@ spec: value: {{ .Values.csi.enableSnapshotter | quote }} - name: CSI_PLUGIN_PRIORITY_CLASSNAME value: {{ .Values.csi.pluginPriorityClassName | quote }} + - name: CSI_PROVISIONER_PRIORITY_CLASSNAME + value: {{ .Values.csi.provisionerPriorityClassName | quote }} {{- if .Values.csi.cephFSPluginUpdateStrategy }} - name: CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY value: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }} diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 60fe87330..cf8732b93 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -60,6 +60,10 @@ csi: enableSnapshotter: true # (Optional) set user created priorityclassName for csi plugin pods. # pluginPriorityClassName: system-node-critical + + # (Optional) set user created priorityclassName for csi provisioner pods. + # provisionerPriorityClassName: system-cluster-critical + # Set logging level for csi containers. # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. #logLevel: 0 diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index 1962c7439..14a7c01da 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -15,6 +15,9 @@ spec: contains: csi-cephfsplugin-metrics spec: serviceAccount: rook-csi-cephfs-provisioner-sa + {{ if .ProvisionerPriorityClassName }} + priorityClassName: {{ .ProvisionerPriorityClassName }} + {{ end }} containers: - name: csi-attacher image: {{ .AttacherImage }} diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml index cd79e0e8a..cb0f8e9e2 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml @@ -16,6 +16,9 @@ spec: contains: csi-cephfsplugin-metrics spec: serviceAccount: rook-csi-cephfs-provisioner-sa + {{ if .ProvisionerPriorityClassName }} + priorityClassName: {{ .ProvisionerPriorityClassName }} + {{ end }} containers: - name: csi-attacher image: {{ .AttacherImage }} diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 91335140a..625838b45 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -15,6 +15,9 @@ spec: contains: csi-rbdplugin-metrics spec: serviceAccount: rook-csi-rbd-provisioner-sa + {{ if .ProvisionerPriorityClassName }} + priorityClassName: {{ .ProvisionerPriorityClassName }} + {{ end }} containers: - name: csi-provisioner image: {{ .ProvisionerImage }} diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml index f015bcf6f..11bf4aecf 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml @@ -16,6 +16,9 @@ spec: contains: csi-rbdplugin-metrics spec: serviceAccount: rook-csi-rbd-provisioner-sa + {{ if .ProvisionerPriorityClassName }} + priorityClassName: {{ .ProvisionerPriorityClassName }} + {{ end }} containers: - name: csi-provisioner image: {{ .ProvisionerImage }} diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 64c50d818..92e9bfc35 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -132,6 +132,9 @@ data: # (Optional) set user created priorityclassName for csi plugin pods. # CSI_PLUGIN_PRIORITY_CLASSNAME="system-node-critical" + # (Optional) set user created priorityclassName for csi provisioner pods. + # CSI_PROVISIONER_PRIORITY_CLASSNAME="system-cluster-critical" + # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index c82eaf0fe..40cfcd067 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -55,6 +55,9 @@ data: # (Optional) set user created priorityclassName for csi plugin pods. # CSI_PLUGIN_PRIORITY_CLASSNAME="system-node-critical" + + # (Optional) set user created priorityclassName for csi provisioner pods. + # CSI_PROVISIONER_PRIORITY_CLASSNAME="system-cluster-critical" # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 790452b6a..022291020 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -40,25 +40,26 @@ import ( ) type Param struct { - CSIPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string - ResizerImage string - DriverNamePrefix string - EnableSnapshotter string - EnableCSIGRPCMetrics string - KubeletDirPath string - ForceCephFSKernelClient string - CephFSPluginUpdateStrategy string - RBDPluginUpdateStrategy string - PluginPriorityClassName string - LogLevel uint8 - CephFSGRPCMetricsPort uint16 - CephFSLivenessMetricsPort uint16 - RBDGRPCMetricsPort uint16 - RBDLivenessMetricsPort uint16 + CSIPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string + ResizerImage string + DriverNamePrefix string + EnableSnapshotter string + EnableCSIGRPCMetrics string + KubeletDirPath string + ForceCephFSKernelClient string + CephFSPluginUpdateStrategy string + RBDPluginUpdateStrategy string + PluginPriorityClassName string + ProvisionerPriorityClassName string + LogLevel uint8 + CephFSGRPCMetricsPort uint16 + CephFSLivenessMetricsPort uint16 + RBDGRPCMetricsPort uint16 + RBDLivenessMetricsPort uint16 } type templateParam struct { @@ -280,6 +281,13 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers return errors.Wrap(err, "failed to load CSI_PLUGIN_PRIORITY_CLASSNAME setting") } + // default value `system-cluster-critical` is applied for some + // critical pods in cluster but less priority than plugin pods + tp.ProvisionerPriorityClassName, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PROVISIONER_PRIORITY_CLASSNAME", "") + if err != nil { + return errors.Wrap(err, "failed to load CSI_PROVISIONER_PRIORITY_CLASSNAME setting") + } + updateStrategy, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate) if err != nil { return errors.Wrap(err, "failed to load CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY setting")