CSI: Make attach mandatory in ceph CSIDriver object

A recent PR #4172 added the ability for rook to create
the CSIDriver object on a kubernetes cluster. The change also
made it so that attachRequired value in the ceph CSI drivers
were set to false.

This leads to the entire attach skipping the AD controller
that deals with RWO attachment enforcement. Thus, if a
CephFS volume is marked RWO, 2 pods on different hosts can
end up mounting the same and consuming it at the same time.

Further for RBD, currently the image watchers are the only ones
that prevent a double mount of an image, other than the AD controller
enforcement of single attach for RWO RBD PVs. This has proven
not to be reliable in the face of MON failures, as the watcher
information is lost, and a second mount is allowed.

Thus, we need to bring back the attach requirement to leverage
kubernetes checks and balances for RWO volume type, and
prevent unwanted multi-attach use of the same.

This PR hence undoes some of the changes introduced by the
PR #4172, to bring the attacher side car back, and also declare
the attachRequired for both RBD and CephFS CSI plugins as true.

NOTE: Initially the attachRequired was set to false as this
improved attach times considerably on kubernetes, as the
entire phase was being avoided.

Signed-off-by: ShyamsundarR <srangana@redhat.com>

Documentation/ceph-upgrade.md

@@ -293,7 +293,6 @@ below, which you should change to match where your images are located.
         value: "quay.io/k8scsi/csi-provisioner:v1.3.0"
     - name: ROOK_CSI_SNAPSHOTTER_IMAGE
         value: "quay.io/k8scsi/csi-snapshotter:v1.2.0"
-    #ROOK_CSI_ATTACHER_IMAGE is required if Kubernetes version is 1.13.x
     - name: ROOK_CSI_ATTACHER_IMAGE
         value: "quay.io/k8scsi/csi-attacher:v1.2.0"
 ```

cluster/charts/rook-ceph/values.yaml

@@ -81,7 +81,6 @@ csi:
     #image: quay.io/k8scsi/csi-provisioner:v1.3.0
   #snapshotter:
     #image: quay.io/k8scsi/csi-snapshotter:v1.2.0
-  # attacher is required if Kubernetes version is 1.13.x
   #attacher:
     #image: quay.io/k8scsi/csi-attacher:v1.2.0
 

cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml

@@ -16,6 +16,22 @@ spec:
     spec:
       serviceAccount: rook-csi-cephfs-provisioner-sa
       containers:
+        - name: csi-attacher
+          image: {{ .AttacherImage }}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=true"
+            - "--timeout=150s"
+            - "--leader-election-type=leases"
+            - "--leader-election-namespace={{ .Namespace }}"
+          env:
+            - name: ADDRESS
+              value: /csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
         - name: csi-provisioner
           image: {{ .ProvisionerImage }}
           args:

cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml

@@ -33,6 +33,22 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+        - name: csi-rbdplugin-attacher
+          image: {{ .AttacherImage }}
+          args:
+            - "--v=5"
+            - "--timeout=150s"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=true"
+            - "--leader-election-type=leases"
+            - "--leader-election-namespace={{ .Namespace }}"
+          env:
+            - name: ADDRESS
+              value: /csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
         - name: csi-snapshotter
           image:  {{ .SnapshotterImage }}
           args:

cluster/examples/kubernetes/ceph/operator.yaml

@@ -179,7 +179,6 @@ spec:
         #  value: "quay.io/k8scsi/csi-provisioner:v1.3.0"
         #- name: ROOK_CSI_SNAPSHOTTER_IMAGE
         #  value: "quay.io/k8scsi/csi-snapshotter:v1.2.0"
-        # ROOK_CSI_ATTACHER_IMAGE is required if Kubernetes version is 1.13.x
         #- name: ROOK_CSI_ATTACHER_IMAGE
         #  value: "quay.io/k8scsi/csi-attacher:v1.2.0"
         # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path.

pkg/operator/ceph/csi/spec.go

@@ -330,7 +330,7 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers
 
 // createCSIDriverInfo Registers CSI driver by creating a CSIDriver object
 func createCSIDriverInfo(clientset kubernetes.Interface, name string) error {
-	attach := false
+	attach := true
 	mountInfo := false
 	// Create CSIDriver object
 	csiDriver := &k8scsi.CSIDriver{