From c6e48823af4ac1cbf7930f548b14840fc940dcaf Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 14 Mar 2023 06:35:35 +0100 Subject: [PATCH] nfs: add csi-attacher to NFS provisioner Add attacher sidecar to NFS provisioner pod to avoid mounting RWO volumes on two nodes details: https://github.com/rook/rook/issues/10692 Signed-off-by: Madhu Rajanna (cherry picked from commit 4bdab21db0c56bada9671f683363b51fb2ef43fd) --- .../rook-ceph/templates/clusterrole.yaml | 6 ++++++ deploy/charts/rook-ceph/values.yaml | 8 ++++++++ deploy/examples/csi/nfs/rbac.yaml | 6 ++++++ deploy/examples/operator-openshift.yaml | 8 ++++++++ deploy/examples/operator.yaml | 8 ++++++++ pkg/operator/ceph/csi/spec.go | 2 +- .../nfs/csi-nfsplugin-provisioner-dep.yaml | 18 ++++++++++++++++++ 7 files changed, 55 insertions(+), 1 deletion(-) diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml index a766e44e5..830798983 100644 --- a/deploy/charts/rook-ceph/templates/clusterrole.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrole.yaml @@ -441,6 +441,12 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] --- # TODO: remove this, once https://github.com/rook/rook/issues/10141 # is resolved. diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 624adac06..dd42c403f 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -378,6 +378,14 @@ csi: limits: memory: 1Gi cpu: 500m + - name : csi-attacher + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 500m # -- CEPH CSI NFS plugin resource requirement list # @default -- see values.yaml diff --git a/deploy/examples/csi/nfs/rbac.yaml b/deploy/examples/csi/nfs/rbac.yaml index 5acf98a13..029ec4458 100644 --- a/deploy/examples/csi/nfs/rbac.yaml +++ b/deploy/examples/csi/nfs/rbac.yaml @@ -61,6 +61,12 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] --- # TODO: remove this, once https://github.com/rook/rook/issues/10141 # is resolved. diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 5e3dcdc66..00e7be42b 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -517,6 +517,14 @@ data: # limits: # memory: 1Gi # cpu: 500m + # - name : csi-attacher + # resource: + # requests: + # memory: 128Mi + # cpu: 100m + # limits: + # memory: 256Mi + # cpu: 200m # (Optional) CEPH CSI NFS plugin resource requirement list, Put here list of resource # requests and limits you want to apply for plugin pod # CSI_NFS_PLUGIN_RESOURCE: | diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index ffc78bda7..67de560f5 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -445,6 +445,14 @@ data: # limits: # memory: 1Gi # cpu: 500m + # - name : csi-attacher + # resource: + # requests: + # memory: 128Mi + # cpu: 100m + # limits: + # memory: 256Mi + # cpu: 200m # (Optional) CEPH CSI NFS plugin resource requirement list, Put here list of resource # requests and limits you want to apply for plugin pod # CSI_NFS_PLUGIN_RESOURCE: | diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index cbe26cecd..b062f5580 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -656,7 +656,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI } } if EnableNFS { - err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, NFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), false) + err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, NFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), true) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", NFSDriverName) } diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml index 7cf5961b8..6f112c6c1 100644 --- a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml @@ -21,6 +21,24 @@ spec: priorityClassName: {{ .ProvisionerPriorityClassName }} {{ end }} containers: + - name: csi-attacher + image: {{ .AttacherImage }} + args: + - "--v={{ .SidecarLogLevel }}" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--timeout={{ .GRPCTimeout }}" + - "--leader-election-namespace={{ .Namespace }}" + - "--leader-election-lease-duration=137s" + - "--leader-election-renew-deadline=107s" + - "--leader-election-retry-period=26s" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: {{ .ImagePullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi {{ if .EnableNFSSnapshotter }} - name: csi-snapshotter image: {{ .SnapshotterImage }}