From 7c6e4d71bcdf36e4e6e7b69f47af724cddf4ba00 Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Thu, 25 Aug 2022 12:29:10 +0530 Subject: [PATCH] csi: add holder pod support for nfs This commit adds holder pod support for nfs to be able to work in non-host networking scenario, without being affected by restarts of csi-nfsplugin. Signed-off-by: Rakshith R --- pkg/operator/ceph/csi/cluster_config.go | 11 ++++ pkg/operator/ceph/csi/spec.go | 18 ++++++ .../template/nfs/csi-nfsplugin-holder.yaml | 64 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-holder.yaml diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go index 8cd5a27d8..d00350c54 100644 --- a/pkg/operator/ceph/csi/cluster_config.go +++ b/pkg/operator/ceph/csi/cluster_config.go @@ -42,6 +42,7 @@ type CsiClusterConfigEntry struct { Monitors []string `json:"monitors"` Namespace string `json:"namespace"` CephFS *CsiCephFSSpec `json:"cephFS,omitempty"` + NFS *CsiNFSSpec `json:"nfs,omitempty"` RBD *CsiRBDSpec `json:"rbd,omitempty"` RadosNamespace string `json:"radosNamespace,omitempty"` } @@ -51,6 +52,10 @@ type CsiCephFSSpec struct { SubvolumeGroup string `json:"subvolumeGroup,omitempty"` } +type CsiNFSSpec struct { + NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"` +} + type CsiRBDSpec struct { NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"` RadosNamespace string `json:"radosNamespace,omitempty"` @@ -142,6 +147,9 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") { centry.CephFS = newCsiClusterConfigEntry.CephFS } + if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" { + centry.NFS = newCsiClusterConfigEntry.NFS + } if newCsiClusterConfigEntry.RBD != nil && (newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "") { centry.RBD = newCsiClusterConfigEntry.RBD } @@ -169,6 +177,9 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") { centry.CephFS = newCsiClusterConfigEntry.CephFS } + if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" { + centry.NFS = newCsiClusterConfigEntry.NFS + } cc = append(cc, centry) } } diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 3e2a2ebb3..dc258f241 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -162,6 +162,8 @@ var ( NFSPluginTemplatePath string //go:embed template/nfs/csi-nfsplugin-provisioner-dep.yaml NFSProvisionerDepTemplatePath string + //go:embed template/nfs/csi-nfsplugin-holder.yaml + NFSPluginHolderTemplatePath string ) const ( @@ -240,6 +242,7 @@ const ( RBDDriverShortName = "rbd" CephFSDriverShortName = "cephfs" + NFSDriverShortName = "nfs" rbdDriverSuffix = "rbd.csi.ceph.com" cephFSDriverSuffix = "cephfs.csi.ceph.com" ) @@ -375,6 +378,14 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if err != nil { return errors.Wrap(err, "failed to load nfs provisioner deployment template") } + enabledDrivers = append(enabledDrivers, driverDetails{ + name: NFSDriverShortName, + fullName: NFSDriverName, + holderTemplate: NFSPluginHolderTemplatePath, + nodeAffinity: nfsPluginNodeAffinityEnv, + toleration: nfsPluginTolerationsEnv, + resource: nfsPluginResource, + }) } holderEnabled := !CSIParam.EnableCSIHostNetwork @@ -561,7 +572,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI } if holderEnabled { nfsPlugin.Spec.Template.Spec.HostNetwork = false + // HostPID is used to communicate with the network namespace + nfsPlugin.Spec.Template.Spec.HostPID = true } + err = k8sutil.CreateDaemonSet(r.opManagerContext, r.opConfig.OperatorNamespace, r.context.Clientset, nfsPlugin) if err != nil { return errors.Wrapf(err, "failed to start nfs plugin daemonset %q", nfsPlugin.Name) @@ -832,6 +846,7 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp Monitors: MonEndpoints(c.clusterInfo.Monitors), RBD: &CsiRBDSpec{}, CephFS: &CsiCephFSSpec{}, + NFS: &CsiNFSSpec{}, } netNamespaceFilePath := generateNetNamespaceFilePath(CSIParam.KubeletDirPath, driver.fullName, c.cluster.Namespace) if driver.name == RBDDriverShortName { @@ -840,6 +855,9 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp if driver.name == CephFSDriverShortName { clusterConfigEntry.CephFS.NetNamespaceFilePath = netNamespaceFilePath } + if driver.name == NFSDriverShortName { + clusterConfigEntry.NFS.NetNamespaceFilePath = netNamespaceFilePath + } // Save the path of the network namespace file for ceph-csi to use err = SaveClusterConfig(r.context.Clientset, c.cluster.Namespace, c.clusterInfo, clusterConfigEntry) if err != nil { diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-holder.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-holder.yaml new file mode 100644 index 000000000..5ee0e6f06 --- /dev/null +++ b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-holder.yaml @@ -0,0 +1,64 @@ +# This DaemonSet is only active when the cluster is deployed with Multus enabled or EnableCSIHostNetwork +# is set to false. Hopefully in the future it will be part of every Ceph-CSI deployment. Its role is to +# "hold" the Ceph Filesystem mounts that are created by the CSI driver. Essentially, it exposes its +# network namespace into the Kubelet plugin directly and Ceph-CSI picks it up. When mounting, the Ceph-CSI +# driver will use the network namespace of the DaemonSet instead of its own (the plugin pod). This pod +# is not expected to be updated nor restarted unless the node reboots. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-nfsplugin-holder + namespace: {{ .Namespace }} +spec: + selector: + matchLabels: + app: csi-nfsplugin-holder + updateStrategy: + # DO NOT CHANGE THE STRATEGY + # The pod is not supposed to be restarted on updates, it should never receive updates too. + # If it does then the connectivity to the Ceph cluster will be lost and the client I/O will + # fail. + type: OnDelete + template: + metadata: + labels: + app: csi-nfsplugin-holder + {{ range $key, $value := .CSINFSPodLabels }} + {{ $key }}: "{{ $value }}" + {{ end }} + spec: + # HostPID is needed to expose the correct process ID network namespace and not the process namespace + hostPID: true + serviceAccountName: rook-csi-nfs-plugin-sa + {{ if .PluginPriorityClassName }} + priorityClassName: {{ .PluginPriorityClassName }} + {{ end }} + containers: + - name: csi-nfsplugin-holder + # This is necessary for the Bidirectional mount propagation + securityContext: + privileged: true + image: {{ .CSIPluginImage }} + command: + - "/bin/sh" + - "-c" + # the third $ allows the "$$" to be interpreted so it can return the current bash process ID + args: + - "ln + --force + --symbolic + --verbose + /proc/$$$/ns/net + {{ .KubeletDirPath }}/plugins/$ROOK_CEPH_CSI_DRIVER_NAME/$CEPH_CLUSTER_NAMESPACE.net.ns + && + sleep + infinity" + volumeMounts: + - name: plugin-mount-dir + mountPath: "{{ .KubeletDirPath }}/plugins" + mountPropagation: "Bidirectional" + volumes: + - name: plugin-mount-dir + hostPath: + path: "{{ .KubeletDirPath }}/plugins" + type: Directory