csi: add holder pod support for nfs

This commit adds holder pod support
for nfs to be able to work in non-host
networking scenario, without being
affected by restarts of csi-nfsplugin.

Signed-off-by: Rakshith R <rar@redhat.com>

pkg/operator/ceph/csi/cluster_config.go

@@ -42,6 +42,7 @@ type CsiClusterConfigEntry struct {
 	Monitors       []string       `json:"monitors"`
 	Namespace      string         `json:"namespace"`
 	CephFS         *CsiCephFSSpec `json:"cephFS,omitempty"`
+	NFS            *CsiNFSSpec    `json:"nfs,omitempty"`
 	RBD            *CsiRBDSpec    `json:"rbd,omitempty"`
 	RadosNamespace string         `json:"radosNamespace,omitempty"`
 }
@@ -51,6 +52,10 @@ type CsiCephFSSpec struct {
 	SubvolumeGroup       string `json:"subvolumeGroup,omitempty"`
 }
 
+type CsiNFSSpec struct {
+	NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"`
+}
+
 type CsiRBDSpec struct {
 	NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"`
 	RadosNamespace       string `json:"radosNamespace,omitempty"`
@@ -142,6 +147,9 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C
 			if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") {
 				centry.CephFS = newCsiClusterConfigEntry.CephFS
 			}
+			if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
+				centry.NFS = newCsiClusterConfigEntry.NFS
+			}
 			if newCsiClusterConfigEntry.RBD != nil && (newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "") {
 				centry.RBD = newCsiClusterConfigEntry.RBD
 			}
@@ -169,6 +177,9 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C
 			if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") {
 				centry.CephFS = newCsiClusterConfigEntry.CephFS
 			}
+			if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
+				centry.NFS = newCsiClusterConfigEntry.NFS
+			}
 			cc = append(cc, centry)
 		}
 	}

pkg/operator/ceph/csi/spec.go

@@ -162,6 +162,8 @@ var (
 	NFSPluginTemplatePath string
 	//go:embed template/nfs/csi-nfsplugin-provisioner-dep.yaml
 	NFSProvisionerDepTemplatePath string
+	//go:embed template/nfs/csi-nfsplugin-holder.yaml
+	NFSPluginHolderTemplatePath string
 )
 
 const (
@@ -240,6 +242,7 @@ const (
 
 	RBDDriverShortName    = "rbd"
 	CephFSDriverShortName = "cephfs"
+	NFSDriverShortName    = "nfs"
 	rbdDriverSuffix       = "rbd.csi.ceph.com"
 	cephFSDriverSuffix    = "cephfs.csi.ceph.com"
 )
@@ -375,6 +378,14 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if err != nil {
 			return errors.Wrap(err, "failed to load nfs provisioner deployment template")
 		}
+		enabledDrivers = append(enabledDrivers, driverDetails{
+			name:           NFSDriverShortName,
+			fullName:       NFSDriverName,
+			holderTemplate: NFSPluginHolderTemplatePath,
+			nodeAffinity:   nfsPluginNodeAffinityEnv,
+			toleration:     nfsPluginTolerationsEnv,
+			resource:       nfsPluginResource,
+		})
 	}
 
 	holderEnabled := !CSIParam.EnableCSIHostNetwork
@@ -561,7 +572,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		}
 		if holderEnabled {
 			nfsPlugin.Spec.Template.Spec.HostNetwork = false
+			// HostPID is used to communicate with the network namespace
+			nfsPlugin.Spec.Template.Spec.HostPID = true
 		}
+
 		err = k8sutil.CreateDaemonSet(r.opManagerContext, r.opConfig.OperatorNamespace, r.context.Clientset, nfsPlugin)
 		if err != nil {
 			return errors.Wrapf(err, "failed to start nfs plugin daemonset %q", nfsPlugin.Name)
@@ -832,6 +846,7 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp
 		Monitors: MonEndpoints(c.clusterInfo.Monitors),
 		RBD:      &CsiRBDSpec{},
 		CephFS:   &CsiCephFSSpec{},
+		NFS:      &CsiNFSSpec{},
 	}
 	netNamespaceFilePath := generateNetNamespaceFilePath(CSIParam.KubeletDirPath, driver.fullName, c.cluster.Namespace)
 	if driver.name == RBDDriverShortName {
@@ -840,6 +855,9 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp
 	if driver.name == CephFSDriverShortName {
 		clusterConfigEntry.CephFS.NetNamespaceFilePath = netNamespaceFilePath
 	}
+	if driver.name == NFSDriverShortName {
+		clusterConfigEntry.NFS.NetNamespaceFilePath = netNamespaceFilePath
+	}
 	// Save the path of the network namespace file for ceph-csi to use
 	err = SaveClusterConfig(r.context.Clientset, c.cluster.Namespace, c.clusterInfo, clusterConfigEntry)
 	if err != nil {

pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-holder.yaml

@@ -0,0 +1,64 @@
+# This DaemonSet is only active when the cluster is deployed with Multus enabled or EnableCSIHostNetwork
+# is set to false. Hopefully in the future it will be part of every Ceph-CSI deployment. Its role is to
+# "hold" the Ceph Filesystem mounts that are created by the CSI driver. Essentially, it exposes its
+# network namespace into the Kubelet plugin directly and Ceph-CSI picks it up. When mounting, the Ceph-CSI
+# driver will use the network namespace of the DaemonSet instead of its own (the plugin pod). This pod
+# is not expected to be updated nor restarted unless the node reboots.
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-nfsplugin-holder
+  namespace: {{ .Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: csi-nfsplugin-holder
+  updateStrategy:
+    # DO NOT CHANGE THE STRATEGY
+    # The pod is not supposed to be restarted on updates, it should never receive updates too.
+    # If it does then the connectivity to the Ceph cluster will be lost and the client I/O will
+    # fail.
+    type: OnDelete
+  template:
+    metadata:
+      labels:
+        app: csi-nfsplugin-holder
+        {{ range $key, $value := .CSINFSPodLabels }}
+        {{ $key }}: "{{ $value }}"
+        {{ end }}
+    spec:
+      # HostPID is needed to expose the correct process ID network namespace and not the process namespace
+      hostPID: true
+      serviceAccountName: rook-csi-nfs-plugin-sa
+      {{ if .PluginPriorityClassName }}
+      priorityClassName: {{ .PluginPriorityClassName }}
+      {{ end }}
+      containers:
+        - name: csi-nfsplugin-holder
+          # This is necessary for the Bidirectional mount propagation
+          securityContext:
+            privileged: true
+          image: {{ .CSIPluginImage }}
+          command:
+            - "/bin/sh"
+            - "-c"
+          # the third $ allows the "$$" to be interpreted so it can return the current bash process ID
+          args:
+            - "ln
+              --force
+              --symbolic
+              --verbose
+              /proc/$$$/ns/net
+              {{ .KubeletDirPath }}/plugins/$ROOK_CEPH_CSI_DRIVER_NAME/$CEPH_CLUSTER_NAMESPACE.net.ns
+              &&
+              sleep
+              infinity"
+          volumeMounts:
+            - name: plugin-mount-dir
+              mountPath: "{{ .KubeletDirPath }}/plugins"
+              mountPropagation: "Bidirectional"
+      volumes:
+        - name: plugin-mount-dir
+          hostPath:
+            path: "{{ .KubeletDirPath }}/plugins"
+            type: Directory