diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go index a552c0a19..7775b0e9c 100644 --- a/pkg/operator/ceph/csi/cluster_config.go +++ b/pkg/operator/ceph/csi/cluster_config.go @@ -103,6 +103,49 @@ func MonEndpoints(mons map[string]*cephclient.MonInfo, requireMsgr2 bool) []stri return endpoints } +// updateNetNamespaceFilePath modify the netNamespaceFilePath for all cluster IDs. +// If holderEnabled is set to true. Otherwise, removes the netNamespaceFilePath value +// for all the clusterIDs. +func updateNetNamespaceFilePath(clusterNamespace string, cc csiClusterConfig) { + var ( + cephFSNetNamespaceFilePath string + rbdNetNamespaceFilePath string + nfsNetNamespaceFilePath string + ) + + if IsHolderEnabled() { + for _, centry := range cc { + if centry.Namespace == clusterNamespace && centry.ClusterID == clusterNamespace { + if centry.CephFS.NetNamespaceFilePath != "" { + cephFSNetNamespaceFilePath = centry.CephFS.NetNamespaceFilePath + } + if centry.RBD.NetNamespaceFilePath != "" { + rbdNetNamespaceFilePath = centry.RBD.NetNamespaceFilePath + } + if centry.NFS.NetNamespaceFilePath != "" { + nfsNetNamespaceFilePath = centry.NFS.NetNamespaceFilePath + } + } + } + + for i, centry := range cc { + if centry.Namespace == clusterNamespace { + cc[i].CephFS.NetNamespaceFilePath = cephFSNetNamespaceFilePath + cc[i].RBD.NetNamespaceFilePath = rbdNetNamespaceFilePath + cc[i].NFS.NetNamespaceFilePath = nfsNetNamespaceFilePath + } + } + } else { + for i := range cc { + if cc[i].Namespace == clusterNamespace { + cc[i].CephFS.NetNamespaceFilePath = "" + cc[i].RBD.NetNamespaceFilePath = "" + cc[i].NFS.NetNamespaceFilePath = "" + } + } + } +} + // updateCsiClusterConfig returns a json-formatted string containing // the cluster-to-mon mapping required to configure ceph csi. func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *CSIClusterConfigEntry) (string, error) { @@ -164,7 +207,7 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C centry.ClusterID = clusterKey centry.Namespace = newCsiClusterConfigEntry.Namespace centry.Monitors = newCsiClusterConfigEntry.Monitors - if newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" { + if newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "" { centry.RBD = newCsiClusterConfigEntry.RBD } // Add a condition not to fill with empty values @@ -181,6 +224,7 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C } } + updateNetNamespaceFilePath(clusterKey, cc) return formatCsiClusterConfig(cc) } diff --git a/pkg/operator/ceph/csi/cluster_config_test.go b/pkg/operator/ceph/csi/cluster_config_test.go index 7d8ac69fc..9a87c39fc 100644 --- a/pkg/operator/ceph/csi/cluster_config_test.go +++ b/pkg/operator/ceph/csi/cluster_config_test.go @@ -18,11 +18,13 @@ package csi import ( "encoding/json" + "reflect" "strings" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/cluster/osd/topology" "github.com/stretchr/testify/assert" cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" @@ -87,6 +89,16 @@ func TestUpdateCsiClusterConfig(t *testing.T) { }, }, } + csiClusterConfigEntry4 := CSIClusterConfigEntry{ + Namespace: "rook-ceph-4", + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: []string{"10.1.1.1:5000"}, + ReadAffinity: cephcsi.ReadAffinity{ + Enabled: true, + CrushLocationLabels: strings.Split(topology.GetDefaultTopologyLabels(), ","), + }, + }, + } var want, s string var err error @@ -376,6 +388,14 @@ func TestUpdateCsiClusterConfig(t *testing.T) { assert.NoError(t, err) compareJSON(t, `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"rbd":{"netNamespaceFilePath":"/var/run/netns/rook-ceph-1","radosNamespace":"rook-ceph-1"},"namespace":"rook-ceph-1"}]`, s) }) + + t.Run("test crush location labels are set", func(t *testing.T) { + s, err = updateCsiClusterConfig("[]", "rook-ceph-4", &csiClusterConfigEntry4) + assert.NoError(t, err) + compareJSON(t, `[{"clusterID":"rook-ceph-4","monitors":["10.1.1.1:5000"],"readAffinity": {"enabled": true, "crushLocationLabels":["kubernetes.io/hostname", + "topology.kubernetes.io/region","topology.kubernetes.io/zone","topology.rook.io/chassis","topology.rook.io/rack","topology.rook.io/row","topology.rook.io/pdu", + "topology.rook.io/pod","topology.rook.io/room","topology.rook.io/datacenter"]},"namespace":"rook-ceph-4"}]`, s) + }) } func contains(src, dest []string) bool { @@ -607,3 +627,232 @@ func TestUpdateCSIDriverOptions(t *testing.T) { }) } } + +func TestUpdateNetNamespaceFilePath(t *testing.T) { + type args struct { + clusterConfig csiClusterConfig + clusterKey string + holderEnabled bool + } + + tests := []struct { + name string + args args + want csiClusterConfig + }{ + { + name: "empty", + args: args{ + clusterKey: "rook-ceph", + holderEnabled: false, + clusterConfig: []CSIClusterConfigEntry{}, + }, + want: []CSIClusterConfigEntry{}, + }, + { + name: "holder enabled", + args: args{ + clusterKey: "rook-ceph", + holderEnabled: true, + clusterConfig: []CSIClusterConfigEntry{ + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + CephFS: cephcsi.CephFS{ + NetNamespaceFilePath: "cephfs.net.ns", + }, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: "rbd.net.ns", + }, + NFS: cephcsi.NFS{ + NetNamespaceFilePath: "nfs.net.ns", + }, + }, + }, + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "cluster-1", + Monitors: []string{"1.1.1.1"}, + CephFS: cephcsi.CephFS{ + SubvolumeGroup: "csi", + }, + }, + }, + }, + }, + want: []CSIClusterConfigEntry{ + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + CephFS: cephcsi.CephFS{ + NetNamespaceFilePath: "cephfs.net.ns", + }, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: "rbd.net.ns", + }, + NFS: cephcsi.NFS{ + NetNamespaceFilePath: "nfs.net.ns", + }, + }, + }, + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "cluster-1", + Monitors: []string{"1.1.1.1"}, + CephFS: cephcsi.CephFS{ + SubvolumeGroup: "csi", + NetNamespaceFilePath: "cephfs.net.ns", + }, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: "rbd.net.ns", + }, + NFS: cephcsi.NFS{ + NetNamespaceFilePath: "nfs.net.ns", + }, + }, + }, + }, + }, + { + name: "holder disabled", + args: args{ + clusterKey: "rook-ceph", + holderEnabled: false, + clusterConfig: []CSIClusterConfigEntry{ + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + CephFS: cephcsi.CephFS{ + NetNamespaceFilePath: "cephfs.net.ns", + }, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: "rbd.net.ns", + }, + NFS: cephcsi.NFS{ + NetNamespaceFilePath: "nfs.net.ns", + }, + }, + }, + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "cluster-1", + Monitors: []string{"1.1.1.1"}, + RBD: cephcsi.RBD{ + RadosNamespace: "group-1", + }, + }, + }, + }, + }, + want: []CSIClusterConfigEntry{ + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + CephFS: cephcsi.CephFS{ + NetNamespaceFilePath: "", + }, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: "", + }, + NFS: cephcsi.NFS{ + NetNamespaceFilePath: "", + }, + }, + }, + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "cluster-1", + Monitors: []string{"1.1.1.1"}, + CephFS: cephcsi.CephFS{ + NetNamespaceFilePath: "", + }, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: "", + RadosNamespace: "group-1", + }, + NFS: cephcsi.NFS{ + NetNamespaceFilePath: "", + }, + }, + }, + }, + }, + } + + cephfsNsFilePath := "cephfs.net.ns" + rbdNsFilePath := "rbd.net.ns" + nfsNsFilePath := "nfs.net.ns" + + csiConfigMap := csiClusterConfig{ + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + CephFS: cephcsi.CephFS{ + NetNamespaceFilePath: cephfsNsFilePath, + }, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: rbdNsFilePath, + }, + NFS: cephcsi.NFS{ + NetNamespaceFilePath: nfsNsFilePath, + }, + }, + }, + { + Namespace: "rook-ceph", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "svg", + }, + }, + { + Namespace: "default", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + holderEnabled = tt.args.holderEnabled + updateNetNamespaceFilePath("rook-ceph", tt.args.clusterConfig) + assert.True(t, reflect.DeepEqual(tt.args.clusterConfig, tt.want)) + }) + } + + t.Run("Holder enabled and disabled later", func(t *testing.T) { + holderEnabled = true + updateNetNamespaceFilePath("rook-ceph", csiConfigMap) + for _, c := range csiConfigMap { + if c.Namespace == "rook-ceph" { + assert.Equal(t, cephfsNsFilePath, c.CephFS.NetNamespaceFilePath) + assert.Equal(t, rbdNsFilePath, c.RBD.NetNamespaceFilePath) + assert.Equal(t, nfsNsFilePath, c.NFS.NetNamespaceFilePath) + } + } + + holderEnabled = false + updateNetNamespaceFilePath("rook-ceph", csiConfigMap) + for _, c := range csiConfigMap { + if c.Namespace == "rook-ceph" { + assert.Equal(t, "", c.CephFS.NetNamespaceFilePath) + assert.Equal(t, "", c.RBD.NetNamespaceFilePath) + assert.Equal(t, "", c.NFS.NetNamespaceFilePath) + + } + } + }) +} diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 9190f031f..eab50e5dc 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -120,7 +120,7 @@ var ( AllowUnsupported = false CustomCSICephConfigExists = false - //driver names + // driver names CephFSDriverName string NFSDriverName string RBDDriverName string @@ -174,6 +174,8 @@ var ( NFSProvisionerDepTemplatePath string //go:embed template/nfs/csi-nfsplugin-holder.yaml NFSPluginHolderTemplatePath string + + holderEnabled bool ) const ( @@ -274,6 +276,10 @@ func CSIEnabled() bool { return EnableRBD || EnableCephFS || EnableNFS } +func IsHolderEnabled() bool { + return holderEnabled +} + func validateCSIParam() error { if len(CSIParam.CSIPluginImage) == 0 { return errors.New("missing csi rbd plugin image") @@ -391,7 +397,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI }) } - holderEnabled := !CSIParam.EnableCSIHostNetwork + holderEnabled = !CSIParam.EnableCSIHostNetwork for i := range r.clustersWithHolder { if r.clustersWithHolder[i].cluster.Spec.Network.IsMultus() { @@ -749,7 +755,6 @@ func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCS CSIParam.CSIPluginImage, corev1.PullPolicy(CSIParam.ImagePullPolicy), ) - if err != nil { return nil, errors.Wrap(err, "failed to set up ceph CSI version job") } @@ -890,6 +895,7 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp }, }, } + netNamespaceFilePath := generateNetNamespaceFilePath(CSIParam.KubeletDirPath, driver.fullName, c.cluster.Namespace) if driver.name == RBDDriverShortName { clusterConfigEntry.RBD.NetNamespaceFilePath = netNamespaceFilePath @@ -900,6 +906,7 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp if driver.name == NFSDriverShortName { clusterConfigEntry.NFS.NetNamespaceFilePath = netNamespaceFilePath } + // Save the path of the network namespace file for ceph-csi to use err = SaveClusterConfig(r.context.Clientset, c.cluster.Namespace, c.clusterInfo, clusterConfigEntry) if err != nil {