diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 2fa1cfbdb..bb5b3edf4 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -119,6 +119,8 @@ The following tables lists the configurable parameters of the rook-operator char | `discover.nodeAffinity` | The node labels for affinity of `discover-agent` (***) | | | `csi.enableRbdDriver` | Enable Ceph CSI RBD driver. | `true` | | `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver. | `true` | +| `csi.pluginPriorityClassName` | PriorityClassName to be set on csi driver plugin pods. | | +| `csi.provisionerPriorityClassName` | PriorityClassName to be set on csi driver provisioner pods. | | | `csi.logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | | `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics. | `true` | | `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment. | | diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 5ffc90367..717131ca0 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -108,6 +108,10 @@ spec: value: {{ .Values.csi.enableCephfsDriver | quote }} - name: CSI_ENABLE_SNAPSHOTTER value: {{ .Values.csi.enableSnapshotter | quote }} + - name: CSI_PLUGIN_PRIORITY_CLASSNAME + value: {{ .Values.csi.pluginPriorityClassName | quote }} + - name: CSI_PROVISIONER_PRIORITY_CLASSNAME + value: {{ .Values.csi.provisionerPriorityClassName | quote }} {{- if .Values.csi.cephFSPluginUpdateStrategy }} - name: CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY value: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }} diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 4253c13e9..214e87ec6 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -58,6 +58,12 @@ csi: enableCephfsDriver: true enableGrpcMetrics: true enableSnapshotter: true + # (Optional) set user created priorityclassName for csi plugin pods. + # pluginPriorityClassName: system-node-critical + + # (Optional) set user created priorityclassName for csi provisioner pods. + # provisionerPriorityClassName: system-cluster-critical + # Set logging level for csi containers. # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. #logLevel: 0 diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index 1962c7439..14a7c01da 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -15,6 +15,9 @@ spec: contains: csi-cephfsplugin-metrics spec: serviceAccount: rook-csi-cephfs-provisioner-sa + {{ if .ProvisionerPriorityClassName }} + priorityClassName: {{ .ProvisionerPriorityClassName }} + {{ end }} containers: - name: csi-attacher image: {{ .AttacherImage }} diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml index cd79e0e8a..cb0f8e9e2 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml @@ -16,6 +16,9 @@ spec: contains: csi-cephfsplugin-metrics spec: serviceAccount: rook-csi-cephfs-provisioner-sa + {{ if .ProvisionerPriorityClassName }} + priorityClassName: {{ .ProvisionerPriorityClassName }} + {{ end }} containers: - name: csi-attacher image: {{ .AttacherImage }} diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 7217eeeac..77b8aa7dc 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -17,6 +17,9 @@ spec: spec: serviceAccount: rook-csi-cephfs-plugin-sa hostNetwork: true + {{ if .PluginPriorityClassName }} + priorityClassName: {{ .PluginPriorityClassName }} + {{ end }} # to use e.g. Rook orchestrated cluster, and mons' FQDN is # resolved through k8s service, set dns policy to cluster first dnsPolicy: ClusterFirstWithHostNet diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 91335140a..625838b45 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -15,6 +15,9 @@ spec: contains: csi-rbdplugin-metrics spec: serviceAccount: rook-csi-rbd-provisioner-sa + {{ if .ProvisionerPriorityClassName }} + priorityClassName: {{ .ProvisionerPriorityClassName }} + {{ end }} containers: - name: csi-provisioner image: {{ .ProvisionerImage }} diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml index f015bcf6f..11bf4aecf 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml @@ -16,6 +16,9 @@ spec: contains: csi-rbdplugin-metrics spec: serviceAccount: rook-csi-rbd-provisioner-sa + {{ if .ProvisionerPriorityClassName }} + priorityClassName: {{ .ProvisionerPriorityClassName }} + {{ end }} containers: - name: csi-provisioner image: {{ .ProvisionerImage }} diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index be9731578..d0ea98a7c 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -16,6 +16,9 @@ spec: contains: csi-rbdplugin-metrics spec: serviceAccount: rook-csi-rbd-plugin-sa + {{ if .PluginPriorityClassName }} + priorityClassName: {{ .PluginPriorityClassName }} + {{ end }} hostNetwork: true hostPID: true # to use e.g. Rook orchestrated cluster, and mons' FQDN is diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 44781282c..cafc80b60 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -129,6 +129,12 @@ data: # ROOK_CSI_SNAPSHOTTER_IMAGE: "quay.io/k8scsi/csi-snapshotter:v1.2.2" # ROOK_CSI_ATTACHER_IMAGE: "quay.io/k8scsi/csi-attacher:v2.1.0" + # (Optional) set user created priorityclassName for csi plugin pods. + # CSI_PLUGIN_PRIORITY_CLASSNAME="system-node-critical" + + # (Optional) set user created priorityclassName for csi provisioner pods. + # CSI_PROVISIONER_PRIORITY_CLASSNAME="system-cluster-critical" + # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index b735ddcb6..9ed954a73 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -53,6 +53,12 @@ data: # ROOK_CSI_SNAPSHOTTER_IMAGE: "quay.io/k8scsi/csi-snapshotter:v1.2.2" # ROOK_CSI_ATTACHER_IMAGE: "quay.io/k8scsi/csi-attacher:v2.1.0" + # (Optional) set user created priorityclassName for csi plugin pods. + # CSI_PLUGIN_PRIORITY_CLASSNAME="system-node-critical" + + # (Optional) set user created priorityclassName for csi provisioner pods. + # CSI_PROVISIONER_PRIORITY_CLASSNAME="system-cluster-critical" + # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index cdc723784..f79085742 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -40,24 +40,26 @@ import ( ) type Param struct { - CSIPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string - ResizerImage string - DriverNamePrefix string - EnableSnapshotter string - EnableCSIGRPCMetrics string - KubeletDirPath string - ForceCephFSKernelClient string - CephFSPluginUpdateStrategy string - RBDPluginUpdateStrategy string - LogLevel uint8 - CephFSGRPCMetricsPort uint16 - CephFSLivenessMetricsPort uint16 - RBDGRPCMetricsPort uint16 - RBDLivenessMetricsPort uint16 + CSIPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string + ResizerImage string + DriverNamePrefix string + EnableSnapshotter string + EnableCSIGRPCMetrics string + KubeletDirPath string + ForceCephFSKernelClient string + CephFSPluginUpdateStrategy string + RBDPluginUpdateStrategy string + PluginPriorityClassName string + ProvisionerPriorityClassName string + LogLevel uint8 + CephFSGRPCMetricsPort uint16 + CephFSLivenessMetricsPort uint16 + RBDGRPCMetricsPort uint16 + RBDLivenessMetricsPort uint16 } type templateParam struct { @@ -273,6 +275,19 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers tp.EnableSnapshotter = "true" } + // default value `system-node-critical` is the highest available priority + tp.PluginPriorityClassName, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PLUGIN_PRIORITY_CLASSNAME", "") + if err != nil { + return errors.Wrap(err, "failed to load CSI_PLUGIN_PRIORITY_CLASSNAME setting") + } + + // default value `system-cluster-critical` is applied for some + // critical pods in cluster but less priority than plugin pods + tp.ProvisionerPriorityClassName, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PROVISIONER_PRIORITY_CLASSNAME", "") + if err != nil { + return errors.Wrap(err, "failed to load CSI_PROVISIONER_PRIORITY_CLASSNAME setting") + } + updateStrategy, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate) if err != nil { return errors.Wrap(err, "failed to load CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY setting")