diff --git a/go.mod b/go.mod index 3fd8e9a84..6c695b71a 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/ansel1/merry/v2 v2.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect + github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83 github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containernetworking/cni v1.1.2 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect diff --git a/go.sum b/go.sum index 6b84276bd..714dbbe90 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83 h1:xWhLO5MR+diAsZoOcPe0zVe+JcJrqMaVbScShye6pXw= +github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83/go.mod h1:ZSvtS90FCB/becFi/rjy85sSw1igchaWZfUigxN9FxY= github.com/ceph/go-ceph v0.25.0 h1:sorUSVkm0F7tYKrv8afd1eJ8UXwxkYIio9o0xLTX59E= github.com/ceph/go-ceph v0.25.0/go.mod h1:HoEJSH32bMcGzmsqJNmSVeYrrcetSxnMfVnGIXBE59Q= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= diff --git a/pkg/apis/go.mod b/pkg/apis/go.mod index e9d7c85e0..3d9b66672 100644 --- a/pkg/apis/go.mod +++ b/pkg/apis/go.mod @@ -7,13 +7,22 @@ require ( github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1 - github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 ) +require ( + github.com/google/uuid v1.5.0 // indirect + github.com/onsi/gomega v1.30.0 // indirect + golang.org/x/tools v0.16.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/client-go v0.29.0 // indirect + k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + require ( github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/containernetworking/cni v1.1.2 // indirect @@ -30,7 +39,6 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.5.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -52,7 +60,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/gomega v1.30.0 // indirect + github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -63,19 +71,14 @@ require ( golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.16.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/client-go v0.29.0 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231129212854-f0671cc7e66a // indirect - k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) exclude ( diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go index 1153140f1..f0845b82d 100644 --- a/pkg/operator/ceph/cluster/cluster_external.go +++ b/pkg/operator/ceph/cluster/cluster_external.go @@ -20,6 +20,7 @@ package cluster import ( "context" + cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -129,7 +130,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error // Save CSI configmap monEndpoints := csi.MonEndpoints(cluster.ClusterInfo.Monitors, cluster.Spec.RequireMsgr2()) - err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: monEndpoints}) + csiConfigEntry := &csi.CSIClusterConfigEntry{ + Namespace: cluster.ClusterInfo.Namespace, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: monEndpoints, + }, + } + err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, csiConfigEntry) if err != nil { return errors.Wrap(err, "failed to update csi cluster config") } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index a5961793a..d2cf989ea 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -49,6 +49,8 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + + cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) const ( @@ -1104,7 +1106,13 @@ func (c *Cluster) saveMonConfig() error { } monEndpoints := csi.MonEndpoints(c.ClusterInfo.Monitors, c.spec.RequireMsgr2()) - if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: monEndpoints}); err != nil { + csiConfigEntry := &csi.CSIClusterConfigEntry{ + Namespace: c.ClusterInfo.Namespace, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: monEndpoints, + }, + } + if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, csiConfigEntry); err != nil { return errors.Wrap(err, "failed to update csi cluster config") } diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go index 97928b9bf..11aaeae34 100644 --- a/pkg/operator/ceph/csi/cluster_config.go +++ b/pkg/operator/ceph/csi/cluster_config.go @@ -33,6 +33,8 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + + cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) var ( @@ -40,40 +42,18 @@ var ( configMutex sync.Mutex ) -type CsiClusterConfigEntry struct { - ClusterID string `json:"clusterID"` - Monitors []string `json:"monitors"` - Namespace string `json:"namespace"` - CephFS *CsiCephFSSpec `json:"cephFS,omitempty"` - NFS *CsiNFSSpec `json:"nfs,omitempty"` - RBD *CsiRBDSpec `json:"rbd,omitempty"` - RadosNamespace string `json:"radosNamespace,omitempty"` - ReadAffinity *cephv1.ReadAffinitySpec `json:"readAffinity,omitempty"` +type CSIClusterConfigEntry struct { + cephcsi.ClusterInfo + Namespace string `json:"namespace"` } -type CsiCephFSSpec struct { - NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"` - SubvolumeGroup string `json:"subvolumeGroup,omitempty"` - KernelMountOptions string `json:"kernelMountOptions,omitempty"` - FuseMountOptions string `json:"fuseMountOptions,omitempty"` -} - -type CsiNFSSpec struct { - NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"` -} - -type CsiRBDSpec struct { - NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"` - RadosNamespace string `json:"radosNamespace,omitempty"` -} - -type csiClusterConfig []CsiClusterConfigEntry +type csiClusterConfig []CSIClusterConfigEntry // FormatCsiClusterConfig returns a json-formatted string containing // the cluster-to-mon mapping required to configure ceph csi. func FormatCsiClusterConfig( - clusterKey string, mons map[string]*cephclient.MonInfo) (string, error) { - + clusterKey string, mons map[string]*cephclient.MonInfo, +) (string, error) { cc := make(csiClusterConfig, 1) cc[0].ClusterID = clusterKey cc[0].Monitors = []string{} @@ -125,10 +105,10 @@ func MonEndpoints(mons map[string]*cephclient.MonInfo, requireMsgr2 bool) []stri // updateCsiClusterConfig returns a json-formatted string containing // the cluster-to-mon mapping required to configure ceph csi. -func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *CsiClusterConfigEntry) (string, error) { +func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *CSIClusterConfigEntry) (string, error) { var ( cc csiClusterConfig - centry CsiClusterConfigEntry + centry CSIClusterConfigEntry found bool ) @@ -160,20 +140,15 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C break } centry.Monitors = newCsiClusterConfigEntry.Monitors - if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") { + if newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" { centry.CephFS = newCsiClusterConfigEntry.CephFS } - if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" { + if newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" { centry.NFS = newCsiClusterConfigEntry.NFS } - if newCsiClusterConfigEntry.RBD != nil && (newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "") { + if newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "" { centry.RBD = newCsiClusterConfigEntry.RBD } - // This maintains backward compatibility for existing clusters, from now on the - // preferred way is to use RBD.RadosNamespace - if newCsiClusterConfigEntry.RadosNamespace != "" { - centry.RadosNamespace = newCsiClusterConfigEntry.RadosNamespace - } found = true cc[i] = centry break @@ -186,14 +161,14 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C centry.ClusterID = clusterKey centry.Namespace = newCsiClusterConfigEntry.Namespace centry.Monitors = newCsiClusterConfigEntry.Monitors - if newCsiClusterConfigEntry.RBD != nil && (newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") { + if newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" { centry.RBD = newCsiClusterConfigEntry.RBD } // Add a condition not to fill with empty values - if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") { + if newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" { centry.CephFS = newCsiClusterConfigEntry.CephFS } - if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" { + if newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" { centry.NFS = newCsiClusterConfigEntry.NFS } cc = append(cc, centry) @@ -239,7 +214,7 @@ func CreateCsiConfigMap(ctx context.Context, namespace string, clientset kuberne // value that is provided to ceph-csi uses in the storage class. // The locker l is typically a mutex and is used to prevent the config // map from being updated for multiple clusters simultaneously. -func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, newCsiClusterConfigEntry *CsiClusterConfigEntry) error { +func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, newCsiClusterConfigEntry *CSIClusterConfigEntry) error { // csi is deployed into the same namespace as the operator csiNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) if csiNamespace == "" { @@ -286,7 +261,8 @@ func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string, // updateCSIDriverOptions updates the CSI driver options, including read affinity, kernel mount options // and fuse mount options, for all entries belonging to the same cluster. func updateCSIDriverOptions(curr, clusterKey string, - csiDriverOptions *cephv1.CSIDriverSpec) (string, error) { + csiDriverOptions *cephv1.CSIDriverSpec, +) (string, error) { cc, err := parseCsiClusterConfig(curr) if err != nil { return "", errors.Wrap(err, "failed to parse current csi cluster config") @@ -295,15 +271,13 @@ func updateCSIDriverOptions(curr, clusterKey string, for i := range cc { // If the clusterID belongs to the same cluster, update the entry. if clusterKey == cc[i].Namespace { - cc[i].ReadAffinity = &csiDriverOptions.ReadAffinity - if cc[i].CephFS == nil { - cc[i].CephFS = &CsiCephFSSpec{ - KernelMountOptions: csiDriverOptions.CephFS.KernelMountOptions, - FuseMountOptions: csiDriverOptions.CephFS.FuseMountOptions, - } - } else { - cc[i].CephFS.KernelMountOptions = csiDriverOptions.CephFS.KernelMountOptions - cc[i].CephFS.FuseMountOptions = csiDriverOptions.CephFS.FuseMountOptions + cc[i].ReadAffinity = cephcsi.ReadAffinity{ + Enabled: csiDriverOptions.ReadAffinity.Enabled, + CrushLocationLabels: csiDriverOptions.ReadAffinity.CrushLocationLabels, + } + cc[i].CephFS = cephcsi.CephFS{ + KernelMountOptions: csiDriverOptions.CephFS.KernelMountOptions, + FuseMountOptions: csiDriverOptions.CephFS.FuseMountOptions, } } } diff --git a/pkg/operator/ceph/csi/cluster_config_test.go b/pkg/operator/ceph/csi/cluster_config_test.go index bc406f8df..ad223903e 100644 --- a/pkg/operator/ceph/csi/cluster_config_test.go +++ b/pkg/operator/ceph/csi/cluster_config_test.go @@ -17,46 +17,85 @@ limitations under the License. package csi import ( + "encoding/json" "strings" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/stretchr/testify/assert" + + cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) -func TestUpdateCsiClusterConfig(t *testing.T) { - csiClusterConfigEntry := CsiClusterConfigEntry{ - Namespace: "rook-ceph-1", - Monitors: []string{"1.2.3.4:5000"}, +func unmarshal(s string) ([]CSIClusterConfigEntry, error) { + var csiClusterConfigEntry []CSIClusterConfigEntry + if err := json.Unmarshal([]byte(s), &csiClusterConfigEntry); err != nil { + return csiClusterConfigEntry, err } - csiClusterConfigEntryMultus := CsiClusterConfigEntry{ + + return csiClusterConfigEntry, nil +} + +func compareJSON(t *testing.T, exceptedJSON, actualJSON string) { + var ( + err error + expected, actual []CSIClusterConfigEntry + ) + + expected, err = unmarshal(exceptedJSON) + if err != nil { + t.Error(err) + } + actual, err = unmarshal(exceptedJSON) + if err != nil { + t.Error(err) + } + + assert.Equal(t, expected, actual) +} + +func TestUpdateCsiClusterConfig(t *testing.T) { + csiClusterConfigEntry := CSIClusterConfigEntry{ Namespace: "rook-ceph-1", - Monitors: []string{"1.2.3.4:5000"}, - RBD: &CsiRBDSpec{ - NetNamespaceFilePath: "/var/run/netns/rook-ceph-1", - RadosNamespace: "rook-ceph-1", + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: []string{"1.2.3.4:5000"}, }, } - csiClusterConfigEntry2 := CsiClusterConfigEntry{ - Namespace: "rook-ceph-2", - Monitors: []string{"20.1.1.1:5000", "20.1.1.2:5000", "20.1.1.3:5000"}, + csiClusterConfigEntryMultus := CSIClusterConfigEntry{ + Namespace: "rook-ceph-1", + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: []string{"1.2.3.4:5000"}, + RBD: cephcsi.RBD{ + NetNamespaceFilePath: "/var/run/netns/rook-ceph-1", + RadosNamespace: "rook-ceph-1", + }, + }, } - csiClusterConfigEntry3 := CsiClusterConfigEntry{ + csiClusterConfigEntry2 := CSIClusterConfigEntry{ + Namespace: "rook-ceph-2", + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: []string{"20.1.1.1:5000", "20.1.1.2:5000", "20.1.1.3:5000"}, + }, + } + csiClusterConfigEntry3 := CSIClusterConfigEntry{ Namespace: "rook-ceph-3", - Monitors: []string{"10.1.1.1:5000", "10.1.1.2:5000", "10.1.1.3:5000"}, - CephFS: &CsiCephFSSpec{ - SubvolumeGroup: "my-group", + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: []string{"10.1.1.1:5000", "10.1.1.2:5000", "10.1.1.3:5000"}, + CephFS: cephcsi.CephFS{ + SubvolumeGroup: "my-group", + }, }, } - var s string + var want, s string var err error t.Run("add a simple mons list", func(t *testing.T) { s, err = updateCsiClusterConfig("[]", "rook-ceph-1", &csiClusterConfigEntry) assert.NoError(t, err) - assert.Equal(t, `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph-1"}]`, s) + want = `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph-1"}]` + compareJSON(t, want, s) }) t.Run("add a 2nd mon to the current cluster", func(t *testing.T) { @@ -125,7 +164,6 @@ func TestUpdateCsiClusterConfig(t *testing.T) { assert.Equal(t, "10.1.1.1:5000", cc[2].Monitors[0]) assert.Equal(t, 3, len(cc[2].Monitors)) assert.Equal(t, "my-group", cc[2].CephFS.SubvolumeGroup) - }) t.Run("add a 4th mon to the 3rd cluster and subvolumegroup is preserved", func(t *testing.T) { @@ -149,9 +187,11 @@ func TestUpdateCsiClusterConfig(t *testing.T) { }) t.Run("add subvolumegroup and mons after", func(t *testing.T) { - csiClusterConfigEntry4 := CsiClusterConfigEntry{ - CephFS: &CsiCephFSSpec{ - SubvolumeGroup: "my-group2", + csiClusterConfigEntry4 := CSIClusterConfigEntry{ + ClusterInfo: cephcsi.ClusterInfo{ + CephFS: cephcsi.CephFS{ + SubvolumeGroup: "my-group2", + }, }, } s, err = updateCsiClusterConfig(s, "quatre", &csiClusterConfigEntry4) @@ -181,20 +221,24 @@ func TestUpdateCsiClusterConfig(t *testing.T) { subvolGrpNameofCluster1 := "subvol-group" radosNSofCluster1 := "rados-ns" - csiCluster1ConfigEntry := CsiClusterConfigEntry{ + csiCluster1ConfigEntry := CSIClusterConfigEntry{ Namespace: clusterIDofCluster1, - Monitors: []string{"1.2.3.4:5000"}, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: []string{"1.2.3.4:5000"}, + }, } s, err := updateCsiClusterConfig("[]", clusterIDofCluster1, &csiCluster1ConfigEntry) assert.NoError(t, err) - assert.Equal(t, s, - `[{"clusterID":"rook-ceph","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph"}]`) + want = `[{"clusterID":"rook-ceph","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph"}]` + compareJSON(t, want, s) // add subvolumegroup to same cluster - subVolCsiCluster1Config := CsiClusterConfigEntry{ + subVolCsiCluster1Config := CSIClusterConfigEntry{ Namespace: clusterIDofCluster1, - Monitors: csiCluster1ConfigEntry.Monitors, - CephFS: &CsiCephFSSpec{ - SubvolumeGroup: subvolGrpNameofCluster1, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: csiCluster1ConfigEntry.Monitors, + CephFS: cephcsi.CephFS{ + SubvolumeGroup: subvolGrpNameofCluster1, + }, }, } s, err = updateCsiClusterConfig(s, subvolGrpNameofCluster1, &subVolCsiCluster1Config) @@ -206,12 +250,13 @@ func TestUpdateCsiClusterConfig(t *testing.T) { assert.Equal(t, subvolGrpNameofCluster1, cc[1].CephFS.SubvolumeGroup, cc) // add rados to same cluster - radosNsCsiCluster1Config := CsiClusterConfigEntry{ - Namespace: clusterIDofCluster1, - Monitors: csiCluster1ConfigEntry.Monitors, - RadosNamespace: radosNSofCluster1, - RBD: &CsiRBDSpec{ - RadosNamespace: radosNSofCluster1, + radosNsCsiCluster1Config := CSIClusterConfigEntry{ + Namespace: clusterIDofCluster1, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: csiCluster1ConfigEntry.Monitors, + RBD: cephcsi.RBD{ + RadosNamespace: radosNSofCluster1, + }, }, } s, err = updateCsiClusterConfig(s, radosNSofCluster1, &radosNsCsiCluster1Config) @@ -220,8 +265,6 @@ func TestUpdateCsiClusterConfig(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 3, len(cc), cc) assert.Equal(t, 1, len(cc[2].Monitors)) - // Now the configuration of new entries goes into RBD.RadosNamespace so it should be empty - assert.Empty(t, cc[2].RadosNamespace, cc) assert.Equal(t, radosNSofCluster1, cc[2].RBD.RadosNamespace, cc) // update mon IP's and check is it updating for all clusterID's @@ -241,21 +284,29 @@ func TestUpdateCsiClusterConfig(t *testing.T) { radosNSofCluster2 := "rados-ns-2" cluster2Mons := []string{"192.168.0.2:5000"} - csiCluster2ConfigEntry := CsiClusterConfigEntry{ + csiCluster2ConfigEntry := CSIClusterConfigEntry{ Namespace: clusterIDofCluster2, - Monitors: cluster2Mons, - } - subVolCsiCluster2Config := CsiClusterConfigEntry{ - Namespace: clusterIDofCluster2, - Monitors: cluster2Mons, - CephFS: &CsiCephFSSpec{ - SubvolumeGroup: subvolGrpNameofCluster2, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: cluster2Mons, }, } - radosNsCsiCluster2Config := CsiClusterConfigEntry{ - Namespace: clusterIDofCluster2, - Monitors: cluster2Mons, - RadosNamespace: radosNSofCluster2, + subVolCsiCluster2Config := CSIClusterConfigEntry{ + Namespace: clusterIDofCluster2, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: cluster2Mons, + CephFS: cephcsi.CephFS{ + SubvolumeGroup: subvolGrpNameofCluster2, + }, + }, + } + radosNsCsiCluster2Config := CSIClusterConfigEntry{ + Namespace: clusterIDofCluster2, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: cluster2Mons, + RBD: cephcsi.RBD{ + RadosNamespace: radosNSofCluster2, + }, + }, } s, err = updateCsiClusterConfig(s, clusterIDofCluster2, &csiCluster2ConfigEntry) assert.NoError(t, err) @@ -323,9 +374,8 @@ func TestUpdateCsiClusterConfig(t *testing.T) { t.Run("test multus cluster", func(t *testing.T) { s, err = updateCsiClusterConfig("[]", "rook-ceph-1", &csiClusterConfigEntryMultus) assert.NoError(t, err) - assert.Equal(t, `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph-1","rbd":{"netNamespaceFilePath":"/var/run/netns/rook-ceph-1","radosNamespace":"rook-ceph-1"}}]`, s) + compareJSON(t, `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"rbd":{"netNamespaceFilePath":"/var/run/netns/rook-ceph-1","radosNamespace":"rook-ceph-1"},"namespace":"rook-ceph-1"}]`, s) }) - } func contains(src, dest []string) bool { @@ -408,21 +458,23 @@ func TestUpdateCSIDriverOptions(t *testing.T) { { name: "empty current config", args: args{ - clusterConfig: []CsiClusterConfigEntry{}, + clusterConfig: []CSIClusterConfigEntry{}, clusterKey: "rook-ceph", csiDriverOptions: &cephv1.CSIDriverSpec{}, }, - want: []CsiClusterConfigEntry{}, + want: []CSIClusterConfigEntry{}, wantErr: false, }, { name: "single matching current config", args: args{ - clusterConfig: []CsiClusterConfigEntry{ + clusterConfig: []CSIClusterConfigEntry{ { - ClusterID: "rook-ceph", Namespace: "rook-ceph", - Monitors: []string{"1.1.1.1"}, + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + }, }, }, clusterKey: "rook-ceph", @@ -437,18 +489,20 @@ func TestUpdateCSIDriverOptions(t *testing.T) { }, }, }, - want: []CsiClusterConfigEntry{ + want: []CSIClusterConfigEntry{ { - ClusterID: "rook-ceph", Namespace: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - ReadAffinity: &cephv1.ReadAffinitySpec{ - Enabled: true, - CrushLocationLabels: []string{"topology.rook.io/rack"}, - }, - CephFS: &CsiCephFSSpec{ - KernelMountOptions: "rw,noatime", - FuseMountOptions: "debug", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + ReadAffinity: cephcsi.ReadAffinity{ + Enabled: true, + CrushLocationLabels: []string{"topology.rook.io/rack"}, + }, + CephFS: cephcsi.CephFS{ + KernelMountOptions: "rw,noatime", + FuseMountOptions: "debug", + }, }, }, }, @@ -457,21 +511,27 @@ func TestUpdateCSIDriverOptions(t *testing.T) { { name: "multiple matching current config", args: args{ - clusterConfig: []CsiClusterConfigEntry{ + clusterConfig: []CSIClusterConfigEntry{ { - ClusterID: "rook-ceph", Namespace: "rook-ceph", - Monitors: []string{"1.1.1.1"}, + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + }, }, { - ClusterID: "rook-ceph-2", Namespace: "rook-ceph", - Monitors: []string{"1.1.1.1"}, + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph-2", + Monitors: []string{"1.1.1.1"}, + }, }, { - ClusterID: "rook-ceph-3", Namespace: "rook-ceph-1", - Monitors: []string{"1.1.1.1"}, + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph-3", + Monitors: []string{"1.1.1.1"}, + }, }, }, clusterKey: "rook-ceph", @@ -486,37 +546,43 @@ func TestUpdateCSIDriverOptions(t *testing.T) { }, }, }, - want: []CsiClusterConfigEntry{ + want: []CSIClusterConfigEntry{ { - ClusterID: "rook-ceph", Namespace: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - ReadAffinity: &cephv1.ReadAffinitySpec{ - Enabled: true, - CrushLocationLabels: []string{"topology.rook.io/rack"}, - }, - CephFS: &CsiCephFSSpec{ - KernelMountOptions: "rw,noatime", - FuseMountOptions: "debug", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph", + Monitors: []string{"1.1.1.1"}, + ReadAffinity: cephcsi.ReadAffinity{ + Enabled: true, + CrushLocationLabels: []string{"topology.rook.io/rack"}, + }, + CephFS: cephcsi.CephFS{ + KernelMountOptions: "rw,noatime", + FuseMountOptions: "debug", + }, }, }, { - ClusterID: "rook-ceph-2", Namespace: "rook-ceph", - Monitors: []string{"1.1.1.1"}, - ReadAffinity: &cephv1.ReadAffinitySpec{ - Enabled: true, - CrushLocationLabels: []string{"topology.rook.io/rack"}, - }, - CephFS: &CsiCephFSSpec{ - KernelMountOptions: "rw,noatime", - FuseMountOptions: "debug", + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph-2", + Monitors: []string{"1.1.1.1"}, + ReadAffinity: cephcsi.ReadAffinity{ + Enabled: true, + CrushLocationLabels: []string{"topology.rook.io/rack"}, + }, + CephFS: cephcsi.CephFS{ + KernelMountOptions: "rw,noatime", + FuseMountOptions: "debug", + }, }, }, { - ClusterID: "rook-ceph-3", Namespace: "rook-ceph-1", - Monitors: []string{"1.1.1.1"}, + ClusterInfo: cephcsi.ClusterInfo{ + ClusterID: "rook-ceph-3", + Monitors: []string{"1.1.1.1"}, + }, }, }, wantErr: false, diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 01c0f248b..132b12755 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -36,6 +36,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/version" + + cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) type Param struct { @@ -865,15 +867,20 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp } } - clusterConfigEntry := &CsiClusterConfigEntry{ - Monitors: MonEndpoints(c.clusterInfo.Monitors, c.cluster.Spec.RequireMsgr2()), - RBD: &CsiRBDSpec{}, - CephFS: &CsiCephFSSpec{ - FuseMountOptions: c.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions, - KernelMountOptions: c.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions, + clusterConfigEntry := &CSIClusterConfigEntry{ + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: MonEndpoints(c.clusterInfo.Monitors, c.cluster.Spec.RequireMsgr2()), + RBD: cephcsi.RBD{}, + CephFS: cephcsi.CephFS{ + FuseMountOptions: c.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions, + KernelMountOptions: c.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions, + }, + NFS: cephcsi.NFS{}, + ReadAffinity: cephcsi.ReadAffinity{ + Enabled: c.clusterInfo.CSIDriverSpec.ReadAffinity.Enabled, + CrushLocationLabels: c.clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels, + }, }, - NFS: &CsiNFSSpec{}, - ReadAffinity: &c.clusterInfo.CSIDriverSpec.ReadAffinity, } netNamespaceFilePath := generateNetNamespaceFilePath(CSIParam.KubeletDirPath, driver.fullName, c.cluster.Namespace) if driver.name == RBDDriverShortName { diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index dc9a18066..dd0c49477 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -46,6 +46,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + + cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) const ( @@ -278,15 +280,20 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateClusterConfig(cephFilesyst // Update CSI config map // If the mon endpoints change, the mon health check go routine will take care of updating the // config map, so no special care is needed in this controller - csiClusterConfigEntry := csi.CsiClusterConfigEntry{ + csiClusterConfigEntry := csi.CSIClusterConfigEntry{ Namespace: r.clusterInfo.Namespace, - Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()), - CephFS: &csi.CsiCephFSSpec{ - SubvolumeGroup: getSubvolumeGroupName(cephFilesystemSubVolumeGroup), - KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions, - FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions, + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()), + CephFS: cephcsi.CephFS{ + SubvolumeGroup: getSubvolumeGroupName(cephFilesystemSubVolumeGroup), + KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions, + FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions, + }, + ReadAffinity: cephcsi.ReadAffinity{ + Enabled: r.clusterInfo.CSIDriverSpec.ReadAffinity.Enabled, + CrushLocationLabels: r.clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels, + }, }, - ReadAffinity: &r.clusterInfo.CSIDriverSpec.ReadAffinity, } // If the cluster has Multus enabled we need to append the network namespace of the driver's diff --git a/pkg/operator/ceph/pool/radosnamespace/controller.go b/pkg/operator/ceph/pool/radosnamespace/controller.go index 726aa63f1..064e9cd67 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller.go @@ -44,6 +44,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + + cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes" ) const ( @@ -264,18 +266,22 @@ func (r *ReconcileCephBlockPoolRadosNamespace) updateClusterConfig(cephBlockPool // Update CSI config map // If the mon endpoints change, the mon health check go routine will take care of updating the // config map, so no special care is needed in this controller - csiClusterConfigEntry := csi.CsiClusterConfigEntry{ + csiClusterConfigEntry := csi.CSIClusterConfigEntry{ Namespace: r.clusterInfo.Namespace, - Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()), - RBD: &csi.CsiRBDSpec{ - RadosNamespace: getRadosNamespaceName(cephBlockPoolRadosNamespace), + ClusterInfo: cephcsi.ClusterInfo{ + Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()), + RBD: cephcsi.RBD{ + RadosNamespace: getRadosNamespaceName(cephBlockPoolRadosNamespace), + }, + CephFS: cephcsi.CephFS{ + KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions, + FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions, + }, + ReadAffinity: cephcsi.ReadAffinity{ + Enabled: r.clusterInfo.CSIDriverSpec.ReadAffinity.Enabled, + CrushLocationLabels: r.clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels, + }, }, - RadosNamespace: getRadosNamespaceName(cephBlockPoolRadosNamespace), - CephFS: &csi.CsiCephFSSpec{ - KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions, - FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions, - }, - ReadAffinity: &r.clusterInfo.CSIDriverSpec.ReadAffinity, } if cephCluster.Spec.Network.IsMultus() {