diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index ec825d8dd..e0de08d36 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -70,6 +70,9 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.csiCephFSPluginVolume` | The volume of the CephCSI CephFS plugin DaemonSet | `nil` | | `csi.csiCephFSPluginVolumeMount` | The volume mounts of the CephCSI CephFS plugin DaemonSet | `nil` | | `csi.csiCephFSProvisionerResource` | CEPH CSI CephFS provisioner resource requirement list | see values.yaml | +| `csi.csiLeaderElectionLeaseDuration` | Duration in seconds that non-leader candidates will wait to force acquire leadership. | `137s` | +| `csi.csiLeaderElectionRenewDeadline` | Deadline in seconds that the acting leader will retry refreshing leadership before giving up. | `107s` | +| `csi.csiLeaderElectionRetryPeriod` | Retry period in seconds the LeaderElector clients should wait between tries of actions. | `26s` | | `csi.csiNFSPluginResource` | CEPH CSI NFS plugin resource requirement list | see values.yaml | | `csi.csiNFSProvisionerResource` | CEPH CSI NFS provisioner resource requirement list | see values.yaml | | `csi.csiRBDPluginResource` | CEPH CSI RBD plugin resource requirement list | see values.yaml | diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 2e08a9463..5e68c8d36 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -61,6 +61,15 @@ data: {{- if .Values.csi.kubeletDirPath }} ROOK_CSI_KUBELET_DIR_PATH: {{ .Values.csi.kubeletDirPath | quote }} {{- end }} +{{- if .Values.csi.csiLeaderElectionLeaseDuration }} + CSI_LEADER_ELECTION_LEASE_DURATION: {{ .Values.csi.csiLeaderElectionLeaseDuration | quote }} +{{- end }} +{{- if .Values.csi.csiLeaderElectionRenewDeadline }} + CSI_LEADER_ELECTION_RENEW_DEADLINE: {{ .Values.csi.csiLeaderElectionRenewDeadline | quote }} +{{- end }} +{{- if .Values.csi.csiLeaderElectionRetryPeriod }} + CSI_LEADER_ELECTION_RETRY_PERIOD: {{ .Values.csi.csiLeaderElectionRetryPeriod | quote }} +{{- end }} {{- if .Values.csi.cephcsi }} {{- if .Values.csi.cephcsi.image }} ROOK_CSI_CEPH_IMAGE: {{ .Values.csi.cephcsi.image | quote }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index e06dcc4cd..257767341 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -477,6 +477,18 @@ csi: # @default -- `/var/lib/kubelet` kubeletDirPath: + # -- Duration in seconds that non-leader candidates will wait to force acquire leadership. + # @default -- `137s` + csiLeaderElectionLeaseDuration: + + # -- Deadline in seconds that the acting leader will retry refreshing leadership before giving up. + # @default -- `107s` + csiLeaderElectionRenewDeadline: + + # -- Retry period in seconds the LeaderElector clients should wait between tries of actions. + # @default -- `26s` + csiLeaderElectionRetryPeriod: + cephcsi: # -- Ceph CSI image # @default -- `quay.io/cephcsi/cephcsi:v3.10.1` diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 44c6f6fce..c265681cd 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -604,6 +604,16 @@ data: CSI_CEPHFS_ATTACH_REQUIRED: "true" CSI_RBD_ATTACH_REQUIRED: "true" CSI_NFS_ATTACH_REQUIRED: "true" + + # (Optional) Duration in seconds that non-leader candidates will wait to force acquire leadership. Default to 137 seconds. + # CSI_LEADER_ELECTION_LEASE_DURATION: "137s" + + # (Optional) Deadline in seconds that the acting leader will retry refreshing leadership before giving up. Defaults to 107 seconds. + # CSI_LEADER_ELECTION_RENEW_DEADLINE: "107s" + + # (Optional) Retry period in seconds the LeaderElector clients should wait between tries of actions. Defaults to 26 seconds. + # CSI_LEADER_ELECTION_RETRY_PERIOD: "26s" + # Rook Discover toleration. Will tolerate all taints with all keys. # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. # DISCOVER_TOLERATIONS: | diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 9c9204cb7..3839ed32a 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -487,6 +487,15 @@ data: # Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR # CSI_CEPHFS_KERNEL_MOUNT_OPTIONS: "ms_mode=secure" + # (Optional) Duration in seconds that non-leader candidates will wait to force acquire leadership. Default to 137 seconds. + # CSI_LEADER_ELECTION_LEASE_DURATION: "137s" + + # (Optional) Deadline in seconds that the acting leader will retry refreshing leadership before giving up. Defaults to 107 seconds. + # CSI_LEADER_ELECTION_RENEW_DEADLINE: "107s" + + # (Optional) Retry Period in seconds the LeaderElector clients should wait between tries of actions. Defaults to 26 seconds. + # CSI_LEADER_ELECTION_RETRY_PERIOD: "26s" + # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 49d79423b..f4aba9cdd 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -238,6 +238,39 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { } } + leaderElectionLeaseDuration := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_LEASE_DURATION", "") + CSIParam.LeaderElectionLeaseDuration = defaultLeaderElectionLeaseDuration + if leaderElectionLeaseDuration != "" { + d, err := time.ParseDuration(leaderElectionLeaseDuration) + if err != nil { + logger.Errorf("failed to parse CSI_LEADER_ELECTION_LEASE_DURATION. Defaulting to %s. %v", defaultLeaderElectionLeaseDuration, err) + } else { + CSIParam.LeaderElectionLeaseDuration = d + } + } + + leaderElectionRenewDeadline := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_RENEW_DEADLINE", "") + CSIParam.LeaderElectionRenewDeadline = defaultLeaderElectionRenewDeadline + if leaderElectionRenewDeadline != "" { + d, err := time.ParseDuration(leaderElectionRenewDeadline) + if err != nil { + logger.Errorf("failed to parse CSI_LEADER_ELECTION_RENEW_DEADLINE. Defaulting to %s. %v", defaultLeaderElectionRenewDeadline, err) + } else { + CSIParam.LeaderElectionRenewDeadline = d + } + } + + leaderElectionRetryPeriod := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_RETRY_PERIOD", "") + CSIParam.LeaderElectionRetryPeriod = defaultLeaderElectionRetryPeriod + if leaderElectionRetryPeriod != "" { + d, err := time.ParseDuration(leaderElectionRetryPeriod) + if err != nil { + logger.Errorf("failed to parse CSI_LEADER_ELECTION_RETRY_PERIOD. Defaulting to %s. %v", defaultLeaderElectionRetryPeriod, err) + } else { + CSIParam.LeaderElectionRetryPeriod = d + } + } + CSIParam.ProvisionerReplicas = defaultProvisionerReplicas nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{}) if err == nil { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 01c0f248b..4527ff7e6 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -85,6 +85,9 @@ type Param struct { CephFSLivenessMetricsPort uint16 CSIAddonsPort uint16 RBDLivenessMetricsPort uint16 + LeaderElectionLeaseDuration time.Duration + LeaderElectionRenewDeadline time.Duration + LeaderElectionRetryPeriod time.Duration ProvisionerReplicas int32 CSICephFSPodLabels map[string]string CSINFSPodLabels map[string]string @@ -233,6 +236,11 @@ const ( defaultLogLevel uint8 = 0 defaultSidecarLogLevel uint8 = 0 + // default leader election flags + defaultLeaderElectionLeaseDuration = 137 * time.Second + defaultLeaderElectionRenewDeadline = 107 * time.Second + defaultLeaderElectionRetryPeriod = 26 * time.Second + // GRPC timeout. defaultGRPCTimeout = 150 grpcTimeout = "CSI_GRPC_TIMEOUT_SECONDS" diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index af8400591..d280f11f9 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -31,9 +31,9 @@ spec: - "--leader-election=true" - "--timeout={{ .GRPCTimeout }}" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" env: - name: ADDRESS value: /csi/csi-provisioner.sock @@ -51,9 +51,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--extra-create-metadata=true" env: - name: ADDRESS @@ -71,9 +71,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" env: @@ -92,11 +92,11 @@ spec: - "--retry-interval-start=500ms" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" - "--feature-gates=HonorPVReclaimPolicy=true" - "--prevent-volume-mode-conversion=true" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--extra-create-metadata=true" env: - name: ADDRESS @@ -177,9 +177,9 @@ spec: - "--pod-uid=$(POD_UID)" - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" ports: - containerPort: {{ .CSIAddonsPort }} env: diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml index 0dac5e48a..84706dd72 100644 --- a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml @@ -30,9 +30,9 @@ spec: - "--leader-election=true" - "--timeout={{ .GRPCTimeout }}" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" env: - name: ADDRESS value: /csi/csi-provisioner.sock @@ -50,9 +50,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--extra-create-metadata=true" env: - name: ADDRESS @@ -70,9 +70,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" env: @@ -91,11 +91,11 @@ spec: - "--retry-interval-start=500ms" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" - "--feature-gates=HonorPVReclaimPolicy=true" - "--prevent-volume-mode-conversion=true" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 5fbec1759..a564063f1 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -31,9 +31,9 @@ spec: - "--retry-interval-start=500ms" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--default-fstype=ext4" - "--extra-create-metadata=true" - "--prevent-volume-mode-conversion=true" @@ -56,9 +56,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" env: @@ -77,9 +77,9 @@ spec: - "--csi-address=$(ADDRESS)" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--default-fstype=ext4" env: - name: ADDRESS @@ -98,9 +98,9 @@ spec: - "--timeout={{ .GRPCTimeout }}" - "--leader-election=true" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" - "--extra-create-metadata=true" env: - name: ADDRESS @@ -149,9 +149,9 @@ spec: - "--pod-uid=$(POD_UID)" - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" - "--leader-election-namespace={{ .Namespace }}" - - "--leader-election-lease-duration=137s" - - "--leader-election-renew-deadline=107s" - - "--leader-election-retry-period=26s" + - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}" + - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}" + - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}" ports: - containerPort: {{ .CSIAddonsPort }} env: