From d4dcecea65aca68d6f9759ffae4a72409dfb44c4 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 15 Jan 2024 14:25:32 +0100 Subject: [PATCH] csi: make leader election flags configurable This commit adds the flexibility to configure leader election flags as per the user requirement and also keeps the existing values as the default one. resolve: #13475 Signed-off-by: Madhu Rajanna (cherry picked from commit 9b418d24a1681c835be2796d15e2b1274e0f2ef7) --- Documentation/Helm-Charts/operator-chart.md | 3 ++ .../charts/rook-ceph/templates/configmap.yaml | 9 +++++ deploy/charts/rook-ceph/values.yaml | 12 +++++++ deploy/examples/operator-openshift.yaml | 10 ++++++ deploy/examples/operator.yaml | 9 +++++ pkg/operator/ceph/csi/csi.go | 33 +++++++++++++++++++ pkg/operator/ceph/csi/spec.go | 8 +++++ .../csi-cephfsplugin-provisioner-dep.yaml | 30 ++++++++--------- .../nfs/csi-nfsplugin-provisioner-dep.yaml | 24 +++++++------- .../rbd/csi-rbdplugin-provisioner-dep.yaml | 30 ++++++++--------- 10 files changed, 126 insertions(+), 42 deletions(-) diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index ec825d8dd..e0de08d36 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -70,6 +70,9 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.csiCephFSPluginVolume` | The volume of the CephCSI CephFS plugin DaemonSet | `nil` | | `csi.csiCephFSPluginVolumeMount` | The volume mounts of the CephCSI CephFS plugin DaemonSet | `nil` | | `csi.csiCephFSProvisionerResource` | CEPH CSI CephFS provisioner resource requirement list | see values.yaml | +| `csi.csiLeaderElectionLeaseDuration` | Duration in seconds that non-leader candidates will wait to force acquire leadership. | `137s` | +| `csi.csiLeaderElectionRenewDeadline` | Deadline in seconds that the acting leader will retry refreshing leadership before giving up. | `107s` | +| `csi.csiLeaderElectionRetryPeriod` | Retry period in seconds the LeaderElector clients should wait between tries of actions. | `26s` | | `csi.csiNFSPluginResource` | CEPH CSI NFS plugin resource requirement list | see values.yaml | | `csi.csiNFSProvisionerResource` | CEPH CSI NFS provisioner resource requirement list | see values.yaml | | `csi.csiRBDPluginResource` | CEPH CSI RBD plugin resource requirement list | see values.yaml | diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 2e08a9463..5e68c8d36 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -61,6 +61,15 @@ data: {{- if .Values.csi.kubeletDirPath }} ROOK_CSI_KUBELET_DIR_PATH: {{ .Values.csi.kubeletDirPath | quote }} {{- end }} +{{- if .Values.csi.csiLeaderElectionLeaseDuration }} + CSI_LEADER_ELECTION_LEASE_DURATION: {{ .Values.csi.csiLeaderElectionLeaseDuration | quote }} +{{- end }} +{{- if .Values.csi.csiLeaderElectionRenewDeadline }} + CSI_LEADER_ELECTION_RENEW_DEADLINE: {{ .Values.csi.csiLeaderElectionRenewDeadline | quote }} +{{- end }} +{{- if .Values.csi.csiLeaderElectionRetryPeriod }} + CSI_LEADER_ELECTION_RETRY_PERIOD: {{ .Values.csi.csiLeaderElectionRetryPeriod | quote }} +{{- end }} {{- if .Values.csi.cephcsi }} {{- if .Values.csi.cephcsi.image }} ROOK_CSI_CEPH_IMAGE: {{ .Values.csi.cephcsi.image | quote }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index e06dcc4cd..257767341 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -477,6 +477,18 @@ csi: # @default -- `/var/lib/kubelet` kubeletDirPath: + # -- Duration in seconds that non-leader candidates will wait to force acquire leadership. + # @default -- `137s` + csiLeaderElectionLeaseDuration: + + # -- Deadline in seconds that the acting leader will retry refreshing leadership before giving up. + # @default -- `107s` + csiLeaderElectionRenewDeadline: + + # -- Retry period in seconds the LeaderElector clients should wait between tries of actions. + # @default -- `26s` + csiLeaderElectionRetryPeriod: + cephcsi: # -- Ceph CSI image # @default -- `quay.io/cephcsi/cephcsi:v3.10.1` diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 44c6f6fce..c265681cd 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -604,6 +604,16 @@ data: CSI_CEPHFS_ATTACH_REQUIRED: "true" CSI_RBD_ATTACH_REQUIRED: "true" CSI_NFS_ATTACH_REQUIRED: "true" + + # (Optional) Duration in seconds that non-leader candidates will wait to force acquire leadership. Default to 137 seconds. + # CSI_LEADER_ELECTION_LEASE_DURATION: "137s" + + # (Optional) Deadline in seconds that the acting leader will retry refreshing leadership before giving up. Defaults to 107 seconds. + # CSI_LEADER_ELECTION_RENEW_DEADLINE: "107s" + + # (Optional) Retry period in seconds the LeaderElector clients should wait between tries of actions. Defaults to 26 seconds. + # CSI_LEADER_ELECTION_RETRY_PERIOD: "26s" + # Rook Discover toleration. Will tolerate all taints with all keys. # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. # DISCOVER_TOLERATIONS: | diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 9c9204cb7..3839ed32a 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -487,6 +487,15 @@ data: # Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR # CSI_CEPHFS_KERNEL_MOUNT_OPTIONS: "ms_mode=secure" + # (Optional) Duration in seconds that non-leader candidates will wait to force acquire leadership. Default to 137 seconds. + # CSI_LEADER_ELECTION_LEASE_DURATION: "137s" + + # (Optional) Deadline in seconds that the acting leader will retry refreshing leadership before giving up. Defaults to 107 seconds. + # CSI_LEADER_ELECTION_RENEW_DEADLINE: "107s" + + # (Optional) Retry Period in seconds the LeaderElector clients should wait between tries of actions. Defaults to 26 seconds. + # CSI_LEADER_ELECTION_RETRY_PERIOD: "26s" + # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 49d79423b..f4aba9cdd 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -238,6 +238,39 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { } } + leaderElectionLeaseDuration := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_LEASE_DURATION", "") + CSIParam.LeaderElectionLeaseDuration = defaultLeaderElectionLeaseDuration + if leaderElectionLeaseDuration != "" { + d, err := time.ParseDuration(leaderElectionLeaseDuration) + if err != nil { + logger.Errorf("failed to parse CSI_LEADER_ELECTION_LEASE_DURATION. Defaulting to %s. %v", defaultLeaderElectionLeaseDuration, err) + } else { + CSIParam.LeaderElectionLeaseDuration = d + } + } + + leaderElectionRenewDeadline := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_RENEW_DEADLINE", "") + CSIParam.LeaderElectionRenewDeadline = defaultLeaderElectionRenewDeadline + if leaderElectionRenewDeadline != "" { + d, err := time.ParseDuration(leaderElectionRenewDeadline) + if err != nil { + logger.Errorf("failed to parse CSI_LEADER_ELECTION_RENEW_DEADLINE. Defaulting to %s. %v", defaultLeaderElectionRenewDeadline, err) + } else { + CSIParam.LeaderElectionRenewDeadline = d + } + } + + leaderElectionRetryPeriod := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_RETRY_PERIOD", "") + CSIParam.LeaderElectionRetryPeriod = defaultLeaderElectionRetryPeriod + if leaderElectionRetryPeriod != "" { + d, err := time.ParseDuration(leaderElectionRetryPeriod) + if err != nil { + logger.Errorf("failed to parse CSI_LEADER_ELECTION_RETRY_PERIOD. Defaulting to %s. %v", defaultLeaderElectionRetryPeriod, err) + } else { + CSIParam.LeaderElectionRetryPeriod = d + } + } + CSIParam.ProvisionerReplicas = defaultProvisionerReplicas nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{}) if err == nil { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 01c0f248b..4527ff7e6 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -85,6 +85,9 @@ type Param struct { CephFSLivenessMetricsPort uint16 CSIAddonsPort uint16 RBDLivenessMetricsPort uint16 + LeaderElectionLeaseDuration time.Duration + LeaderElectionRenewDeadline time.Duration + LeaderElectionRetryPeriod time.Duration ProvisionerReplicas int32 CSICephFSPodLabels map[string]string CSINFSPodLabels map[string]string @@ -233,6 +236,11 @@ const ( defaultLogLevel uint8 = 0 defaultSidecarLogLevel uint8 = 0 + // default leader election flags + defaultLeaderElectionLeaseDuration = 137 * time.Second + defaultLeaderElectionRenewDeadline = 107 * time.Second + defaultLeaderElectionRetryPeriod = 26 * time.Second + // GRPC timeout. defaultGRPCTimeout = 150 grpcTimeout = "CSI_GRPC_TIMEOUT_SECONDS" diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index af8400591..d280f11f9 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -31,9 +31,9 @@ spec: - "--leader-election=true" - "--timeout={{ .GRPCTimeout }}" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" env: - name: ADDRESS value: /csi/csi-provisioner.sock @@ -51,9 +51,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--extra-create-metadata=true" env: - name: ADDRESS @@ -71,9 +71,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" env: @@ -92,11 +92,11 @@ spec: - "--retry-interval-start=500ms" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" - "--feature-gates=HonorPVReclaimPolicy=true" - "--prevent-volume-mode-conversion=true" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--extra-create-metadata=true" env: - name: ADDRESS @@ -177,9 +177,9 @@ spec: - "--pod-uid=$(POD_UID)" - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" ports: - containerPort: {{ .CSIAddonsPort }} env: diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml index 0dac5e48a..84706dd72 100644 --- a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml @@ -30,9 +30,9 @@ spec: - "--leader-election=true" - "--timeout={{ .GRPCTimeout }}" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" env: - name: ADDRESS value: /csi/csi-provisioner.sock @@ -50,9 +50,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--extra-create-metadata=true" env: - name: ADDRESS @@ -70,9 +70,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" env: @@ -91,11 +91,11 @@ spec: - "--retry-interval-start=500ms" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" - "--feature-gates=HonorPVReclaimPolicy=true" - "--prevent-volume-mode-conversion=true" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 5fbec1759..a564063f1 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -31,9 +31,9 @@ spec: - "--retry-interval-start=500ms" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--default-fstype=ext4" - "--extra-create-metadata=true" - "--prevent-volume-mode-conversion=true" @@ -56,9 +56,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" env: @@ -77,9 +77,9 @@ spec: - "--csi-address=$(ADDRESS)" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--default-fstype=ext4" env: - name: ADDRESS @@ -98,9 +98,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--extra-create-metadata=true" env: - name: ADDRESS @@ -149,9 +149,9 @@ spec: - "--pod-uid=$(POD_UID)" - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" ports: - containerPort: {{ .CSIAddonsPort }} env: