From cc819f00df40ed7c4e05f3c1e5ef640942cf71e6 Mon Sep 17 00:00:00 2001 From: karthik-us Date: Mon, 7 Aug 2023 19:14:43 +0530 Subject: [PATCH] csi: add maxUnavailable for cephfs csi daemonset Adding maxUnavailable parameter to the cephfs csi daemonset. The default value of maxUnavailable = 1 can make the upgrade process slower in environments with large number of nodes. Fixes: #12636 Signed-off-by: karthik-us (cherry picked from commit 2675c976d5a9d93fd4d576d6080a1915f70dc1e2) --- Documentation/Helm-Charts/operator-chart.md | 1 + .../charts/rook-ceph/templates/configmap.yaml | 3 + deploy/charts/rook-ceph/values.yaml | 4 + deploy/examples/operator-openshift.yaml | 3 + deploy/examples/operator.yaml | 3 + pkg/operator/ceph/csi/csi.go | 1 + pkg/operator/ceph/csi/spec.go | 105 +++++++++--------- .../csi/template/cephfs/csi-cephfsplugin.yaml | 4 + 8 files changed, 72 insertions(+), 52 deletions(-) diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 324587d68..60a080918 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -58,6 +58,7 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.cephFSFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html | `"File"` | | `csi.cephFSKernelMountOptions` | Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options. Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR | `nil` | | `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` | +| `csi.cephFSPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. | `1` | | `csi.cephcsi.image` | Ceph CSI image | `quay.io/cephcsi/cephcsi:v3.9.0` | | `csi.cephfsGrpcMetricsPort` | CSI CephFS driver GRPC metrics port | `9091` | | `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port | `9081` | diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index cc312f55a..1281c6ca7 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -30,6 +30,9 @@ data: {{- if .Values.csi.cephFSPluginUpdateStrategy }} CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }} {{- end }} +{{- if .Values.csi.cephFSPluginUpdateStrategyMaxUnavailable }} + CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: {{ .Values.csi.cephFSPluginUpdateStrategyMaxUnavailable | quote }} +{{- end }} {{- if .Values.csi.nfsPluginUpdateStrategy }} CSI_NFS_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.nfsPluginUpdateStrategy | quote }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 838921d39..35ce4c967 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -150,6 +150,10 @@ csi: # @default -- `RollingUpdate` cephFSPluginUpdateStrategy: + # -- A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. + # @default -- `1` + cephFSPluginUpdateStrategyMaxUnavailable: + # -- CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate # @default -- `RollingUpdate` nfsPluginUpdateStrategy: diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 2f8346d0d..a3bcbdf46 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -207,6 +207,9 @@ data: # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" + # A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. + # Default value is 1. + # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: "1" # CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_RBD_PLUGIN_UPDATE_STRATEGY: "OnDelete" diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index d0a6bcdb2..513fa7be5 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -126,6 +126,9 @@ data: # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" + # A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. + # Default value is 1. + # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE: "1" # CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. # CSI_RBD_PLUGIN_UPDATE_STRATEGY: "OnDelete" diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 82def04ee..e0a7e0bf2 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -199,6 +199,7 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { CSIParam.CephFSPluginUpdateStrategy = onDelete } else { CSIParam.CephFSPluginUpdateStrategy = rollingUpdate + CSIParam.CephFSPluginUpdateStrategyMaxUnavailable = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") } if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index a562af25a..ce7ef957b 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -40,58 +40,59 @@ import ( ) type Param struct { - CSIPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string - ResizerImage string - DriverNamePrefix string - EnableCSIGRPCMetrics string - KubeletDirPath string - ForceCephFSKernelClient string - CephFSKernelMountOptions string - CephFSPluginUpdateStrategy string - NFSPluginUpdateStrategy string - RBDPluginUpdateStrategy string - RBDPluginUpdateStrategyMaxUnavailable string - PluginPriorityClassName string - ProvisionerPriorityClassName string - VolumeReplicationImage string - CSIAddonsImage string - ImagePullPolicy string - CSIClusterName string - CSIDomainLabels string - CrushLocationLabels string - GRPCTimeout time.Duration - CSIEnableMetadata bool - EnablePluginSelinuxHostMount bool - EnableCSIHostNetwork bool - EnableOMAPGenerator bool - EnableRBDSnapshotter bool - EnableCephFSSnapshotter bool - EnableNFSSnapshotter bool - EnableCSIAddonsSideCar bool - MountCustomCephConf bool - EnableCSIDriverSeLinuxMount bool - EnableCSIEncryption bool - EnableCSITopology bool - EnableLiveness bool - EnableReadAffinity bool - CephFSAttachRequired bool - RBDAttachRequired bool - NFSAttachRequired bool - LogLevel uint8 - SidecarLogLevel uint8 - CephFSGRPCMetricsPort uint16 - CephFSLivenessMetricsPort uint16 - RBDGRPCMetricsPort uint16 - CSIAddonsPort uint16 - RBDLivenessMetricsPort uint16 - ProvisionerReplicas int32 - CSICephFSPodLabels map[string]string - CSINFSPodLabels map[string]string - CSIRBDPodLabels map[string]string + CSIPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string + ResizerImage string + DriverNamePrefix string + EnableCSIGRPCMetrics string + KubeletDirPath string + ForceCephFSKernelClient string + CephFSKernelMountOptions string + CephFSPluginUpdateStrategy string + CephFSPluginUpdateStrategyMaxUnavailable string + NFSPluginUpdateStrategy string + RBDPluginUpdateStrategy string + RBDPluginUpdateStrategyMaxUnavailable string + PluginPriorityClassName string + ProvisionerPriorityClassName string + VolumeReplicationImage string + CSIAddonsImage string + ImagePullPolicy string + CSIClusterName string + CSIDomainLabels string + CrushLocationLabels string + GRPCTimeout time.Duration + CSIEnableMetadata bool + EnablePluginSelinuxHostMount bool + EnableCSIHostNetwork bool + EnableOMAPGenerator bool + EnableRBDSnapshotter bool + EnableCephFSSnapshotter bool + EnableNFSSnapshotter bool + EnableCSIAddonsSideCar bool + MountCustomCephConf bool + EnableCSIDriverSeLinuxMount bool + EnableCSIEncryption bool + EnableCSITopology bool + EnableLiveness bool + EnableReadAffinity bool + CephFSAttachRequired bool + RBDAttachRequired bool + NFSAttachRequired bool + LogLevel uint8 + SidecarLogLevel uint8 + CephFSGRPCMetricsPort uint16 + CephFSLivenessMetricsPort uint16 + RBDGRPCMetricsPort uint16 + CSIAddonsPort uint16 + RBDLivenessMetricsPort uint16 + ProvisionerReplicas int32 + CSICephFSPodLabels map[string]string + CSINFSPodLabels map[string]string + CSIRBDPodLabels map[string]string } type templateParam struct { diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 58c86496b..9ab68697a 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -9,6 +9,10 @@ spec: app: csi-cephfsplugin updateStrategy: type: {{ .CephFSPluginUpdateStrategy }} + {{ if eq .CephFSPluginUpdateStrategy "RollingUpdate" }} + rollingUpdate: + maxUnavailable: {{ .CephFSPluginUpdateStrategyMaxUnavailable }} + {{ end }} template: metadata: labels: