feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #13420 from iPraveenParihar/api/csi-config-map

Browse Source 
csi: use ceph-csi exposed csi config map API
This commit is contained in:
Travis Nielsen
2024-01-16 12:34:11 -07:00
committed by GitHub
parent 2ba90e9072 dfd3d625a8
commit 591ddf2c3d
10 changed files with 263 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1
View File
@@ -51,6 +51,7 @@ require (
github.com/ansel1/merry/v2 v2.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v3 v3.2.2 // indirect
github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containernetworking/cni v1.1.2 // indirect
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
go.sum
+2
View File
@@ -127,6 +127,8 @@ github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r
github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M=
github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83 h1:xWhLO5MR+diAsZoOcPe0zVe+JcJrqMaVbScShye6pXw=
github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83/go.mod h1:ZSvtS90FCB/becFi/rjy85sSw1igchaWZfUigxN9FxY=
github.com/ceph/go-ceph v0.25.0 h1:sorUSVkm0F7tYKrv8afd1eJ8UXwxkYIio9o0xLTX59E=
github.com/ceph/go-ceph v0.25.0/go.mod h1:HoEJSH32bMcGzmsqJNmSVeYrrcetSxnMfVnGIXBE59Q=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
pkg/apis/go.mod
+11 -8
View File
@@ -7,13 +7,22 @@ require (
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1
github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1
github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.8.4
k8s.io/api v0.29.0
k8s.io/apimachinery v0.29.0
)
require (
github.com/google/uuid v1.5.0 // indirect
github.com/onsi/gomega v1.30.0 // indirect
golang.org/x/tools v0.16.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/client-go v0.29.0 // indirect
k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
require (
github.com/cenkalti/backoff/v3 v3.2.2 // indirect
github.com/containernetworking/cni v1.1.2 // indirect
@@ -30,7 +39,6 @@ require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.5.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -52,7 +60,7 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/onsi/gomega v1.30.0 // indirect
github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
@@ -63,19 +71,14 @@ require (
golang.org/x/term v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.16.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/client-go v0.29.0 // indirect
k8s.io/klog/v2 v2.110.1 // indirect
k8s.io/kube-openapi v0.0.0-20231129212854-f0671cc7e66a // indirect
k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
exclude (
pkg/operator/ceph/cluster/cluster_external.go
+8 -1
View File
@@ -20,6 +20,7 @@ package cluster
import (
"context"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
@@ -129,7 +130,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
// Save CSI configmap
monEndpoints := csi.MonEndpoints(cluster.ClusterInfo.Monitors, cluster.Spec.RequireMsgr2())
err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: monEndpoints})
csiConfigEntry := &csi.CSIClusterConfigEntry{
Namespace: cluster.ClusterInfo.Namespace,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: monEndpoints,
},
}
err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, csiConfigEntry)
if err != nil {
return errors.Wrap(err, "failed to update csi cluster config")
}
pkg/operator/ceph/cluster/mon/mon.go
+9 -1
View File
@@ -49,6 +49,8 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
)
const (
@@ -1104,7 +1106,13 @@ func (c *Cluster) saveMonConfig() error {
}
monEndpoints := csi.MonEndpoints(c.ClusterInfo.Monitors, c.spec.RequireMsgr2())
if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: monEndpoints}); err != nil {
csiConfigEntry := &csi.CSIClusterConfigEntry{
Namespace: c.ClusterInfo.Namespace,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: monEndpoints,
},
}
if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, csiConfigEntry); err != nil {
return errors.Wrap(err, "failed to update csi cluster config")
}
pkg/operator/ceph/csi/cluster_config.go
+26 -52
View File
@@ -33,6 +33,8 @@ import (
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
)
var (
@@ -40,40 +42,18 @@ var (
configMutex sync.Mutex
)
type CsiClusterConfigEntry struct {
ClusterID string `json:"clusterID"`
Monitors []string `json:"monitors"`
Namespace string `json:"namespace"`
CephFS *CsiCephFSSpec `json:"cephFS,omitempty"`
NFS *CsiNFSSpec `json:"nfs,omitempty"`
RBD *CsiRBDSpec `json:"rbd,omitempty"`
RadosNamespace string `json:"radosNamespace,omitempty"`
ReadAffinity *cephv1.ReadAffinitySpec `json:"readAffinity,omitempty"`
type CSIClusterConfigEntry struct {
cephcsi.ClusterInfo
Namespace string `json:"namespace"`
}
type CsiCephFSSpec struct {
NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"`
SubvolumeGroup string `json:"subvolumeGroup,omitempty"`
KernelMountOptions string `json:"kernelMountOptions,omitempty"`
FuseMountOptions string `json:"fuseMountOptions,omitempty"`
}
type CsiNFSSpec struct {
NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"`
}
type CsiRBDSpec struct {
NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"`
RadosNamespace string `json:"radosNamespace,omitempty"`
}
type csiClusterConfig []CsiClusterConfigEntry
type csiClusterConfig []CSIClusterConfigEntry
// FormatCsiClusterConfig returns a json-formatted string containing
// the cluster-to-mon mapping required to configure ceph csi.
func FormatCsiClusterConfig(
clusterKey string, mons map[string]*cephclient.MonInfo) (string, error) {
clusterKey string, mons map[string]*cephclient.MonInfo,
) (string, error) {
cc := make(csiClusterConfig, 1)
cc[0].ClusterID = clusterKey
cc[0].Monitors = []string{}
@@ -125,10 +105,10 @@ func MonEndpoints(mons map[string]*cephclient.MonInfo, requireMsgr2 bool) []stri
// updateCsiClusterConfig returns a json-formatted string containing
// the cluster-to-mon mapping required to configure ceph csi.
func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *CsiClusterConfigEntry) (string, error) {
func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *CSIClusterConfigEntry) (string, error) {
var (
cc csiClusterConfig
centry CsiClusterConfigEntry
centry CSIClusterConfigEntry
found bool
)
@@ -160,20 +140,15 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C
break
}
centry.Monitors = newCsiClusterConfigEntry.Monitors
if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") {
if newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" {
centry.CephFS = newCsiClusterConfigEntry.CephFS
}
if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
if newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
centry.NFS = newCsiClusterConfigEntry.NFS
}
if newCsiClusterConfigEntry.RBD != nil && (newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "") {
if newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "" {
centry.RBD = newCsiClusterConfigEntry.RBD
}
// This maintains backward compatibility for existing clusters, from now on the
// preferred way is to use RBD.RadosNamespace
if newCsiClusterConfigEntry.RadosNamespace != "" {
centry.RadosNamespace = newCsiClusterConfigEntry.RadosNamespace
}
found = true
cc[i] = centry
break
@@ -186,14 +161,14 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C
centry.ClusterID = clusterKey
centry.Namespace = newCsiClusterConfigEntry.Namespace
centry.Monitors = newCsiClusterConfigEntry.Monitors
if newCsiClusterConfigEntry.RBD != nil && (newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") {
if newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" {
centry.RBD = newCsiClusterConfigEntry.RBD
}
// Add a condition not to fill with empty values
if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") {
if newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" {
centry.CephFS = newCsiClusterConfigEntry.CephFS
}
if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
if newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
centry.NFS = newCsiClusterConfigEntry.NFS
}
cc = append(cc, centry)
@@ -239,7 +214,7 @@ func CreateCsiConfigMap(ctx context.Context, namespace string, clientset kuberne
// value that is provided to ceph-csi uses in the storage class.
// The locker l is typically a mutex and is used to prevent the config
// map from being updated for multiple clusters simultaneously.
func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, newCsiClusterConfigEntry *CsiClusterConfigEntry) error {
func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, newCsiClusterConfigEntry *CSIClusterConfigEntry) error {
// csi is deployed into the same namespace as the operator
csiNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
if csiNamespace == "" {
@@ -286,7 +261,8 @@ func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string,
// updateCSIDriverOptions updates the CSI driver options, including read affinity, kernel mount options
// and fuse mount options, for all entries belonging to the same cluster.
func updateCSIDriverOptions(curr, clusterKey string,
csiDriverOptions *cephv1.CSIDriverSpec) (string, error) {
csiDriverOptions *cephv1.CSIDriverSpec,
) (string, error) {
cc, err := parseCsiClusterConfig(curr)
if err != nil {
return "", errors.Wrap(err, "failed to parse current csi cluster config")
@@ -295,15 +271,13 @@ func updateCSIDriverOptions(curr, clusterKey string,
for i := range cc {
// If the clusterID belongs to the same cluster, update the entry.
if clusterKey == cc[i].Namespace {
cc[i].ReadAffinity = &csiDriverOptions.ReadAffinity
if cc[i].CephFS == nil {
cc[i].CephFS = &CsiCephFSSpec{
KernelMountOptions: csiDriverOptions.CephFS.KernelMountOptions,
FuseMountOptions: csiDriverOptions.CephFS.FuseMountOptions,
}
} else {
cc[i].CephFS.KernelMountOptions = csiDriverOptions.CephFS.KernelMountOptions
cc[i].CephFS.FuseMountOptions = csiDriverOptions.CephFS.FuseMountOptions
cc[i].ReadAffinity = cephcsi.ReadAffinity{
Enabled: csiDriverOptions.ReadAffinity.Enabled,
CrushLocationLabels: csiDriverOptions.ReadAffinity.CrushLocationLabels,
}
cc[i].CephFS = cephcsi.CephFS{
KernelMountOptions: csiDriverOptions.CephFS.KernelMountOptions,
FuseMountOptions: csiDriverOptions.CephFS.FuseMountOptions,
}
}
}
pkg/operator/ceph/csi/cluster_config_test.go
+161 -95
View File
@@ -17,46 +17,85 @@ limitations under the License.
package csi
import (
"encoding/json"
"strings"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/stretchr/testify/assert"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
)
func TestUpdateCsiClusterConfig(t *testing.T) {
csiClusterConfigEntry := CsiClusterConfigEntry{
Namespace: "rook-ceph-1",
Monitors: []string{"1.2.3.4:5000"},
func unmarshal(s string) ([]CSIClusterConfigEntry, error) {
var csiClusterConfigEntry []CSIClusterConfigEntry
if err := json.Unmarshal([]byte(s), &csiClusterConfigEntry); err != nil {
return csiClusterConfigEntry, err
}
csiClusterConfigEntryMultus := CsiClusterConfigEntry{
return csiClusterConfigEntry, nil
}
func compareJSON(t *testing.T, exceptedJSON, actualJSON string) {
var (
err error
expected, actual []CSIClusterConfigEntry
)
expected, err = unmarshal(exceptedJSON)
if err != nil {
t.Error(err)
}
actual, err = unmarshal(exceptedJSON)
if err != nil {
t.Error(err)
}
assert.Equal(t, expected, actual)
}
func TestUpdateCsiClusterConfig(t *testing.T) {
csiClusterConfigEntry := CSIClusterConfigEntry{
Namespace: "rook-ceph-1",
Monitors: []string{"1.2.3.4:5000"},
RBD: &CsiRBDSpec{
NetNamespaceFilePath: "/var/run/netns/rook-ceph-1",
RadosNamespace: "rook-ceph-1",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"1.2.3.4:5000"},
},
}
csiClusterConfigEntry2 := CsiClusterConfigEntry{
Namespace: "rook-ceph-2",
Monitors: []string{"20.1.1.1:5000", "20.1.1.2:5000", "20.1.1.3:5000"},
csiClusterConfigEntryMultus := CSIClusterConfigEntry{
Namespace: "rook-ceph-1",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"1.2.3.4:5000"},
RBD: cephcsi.RBD{
NetNamespaceFilePath: "/var/run/netns/rook-ceph-1",
RadosNamespace: "rook-ceph-1",
},
},
}
csiClusterConfigEntry3 := CsiClusterConfigEntry{
csiClusterConfigEntry2 := CSIClusterConfigEntry{
Namespace: "rook-ceph-2",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"20.1.1.1:5000", "20.1.1.2:5000", "20.1.1.3:5000"},
},
}
csiClusterConfigEntry3 := CSIClusterConfigEntry{
Namespace: "rook-ceph-3",
Monitors: []string{"10.1.1.1:5000", "10.1.1.2:5000", "10.1.1.3:5000"},
CephFS: &CsiCephFSSpec{
SubvolumeGroup: "my-group",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"10.1.1.1:5000", "10.1.1.2:5000", "10.1.1.3:5000"},
CephFS: cephcsi.CephFS{
SubvolumeGroup: "my-group",
},
},
}
var s string
var want, s string
var err error
t.Run("add a simple mons list", func(t *testing.T) {
s, err = updateCsiClusterConfig("[]", "rook-ceph-1", &csiClusterConfigEntry)
assert.NoError(t, err)
assert.Equal(t, `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph-1"}]`, s)
want = `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph-1"}]`
compareJSON(t, want, s)
})
t.Run("add a 2nd mon to the current cluster", func(t *testing.T) {
@@ -125,7 +164,6 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
assert.Equal(t, "10.1.1.1:5000", cc[2].Monitors[0])
assert.Equal(t, 3, len(cc[2].Monitors))
assert.Equal(t, "my-group", cc[2].CephFS.SubvolumeGroup)
})
t.Run("add a 4th mon to the 3rd cluster and subvolumegroup is preserved", func(t *testing.T) {
@@ -149,9 +187,11 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
})
t.Run("add subvolumegroup and mons after", func(t *testing.T) {
csiClusterConfigEntry4 := CsiClusterConfigEntry{
CephFS: &CsiCephFSSpec{
SubvolumeGroup: "my-group2",
csiClusterConfigEntry4 := CSIClusterConfigEntry{
ClusterInfo: cephcsi.ClusterInfo{
CephFS: cephcsi.CephFS{
SubvolumeGroup: "my-group2",
},
},
}
s, err = updateCsiClusterConfig(s, "quatre", &csiClusterConfigEntry4)
@@ -181,20 +221,24 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
subvolGrpNameofCluster1 := "subvol-group"
radosNSofCluster1 := "rados-ns"
csiCluster1ConfigEntry := CsiClusterConfigEntry{
csiCluster1ConfigEntry := CSIClusterConfigEntry{
Namespace: clusterIDofCluster1,
Monitors: []string{"1.2.3.4:5000"},
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"1.2.3.4:5000"},
},
}
s, err := updateCsiClusterConfig("[]", clusterIDofCluster1, &csiCluster1ConfigEntry)
assert.NoError(t, err)
assert.Equal(t, s,
`[{"clusterID":"rook-ceph","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph"}]`)
want = `[{"clusterID":"rook-ceph","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph"}]`
compareJSON(t, want, s)
// add subvolumegroup to same cluster
subVolCsiCluster1Config := CsiClusterConfigEntry{
subVolCsiCluster1Config := CSIClusterConfigEntry{
Namespace: clusterIDofCluster1,
Monitors: csiCluster1ConfigEntry.Monitors,
CephFS: &CsiCephFSSpec{
SubvolumeGroup: subvolGrpNameofCluster1,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: csiCluster1ConfigEntry.Monitors,
CephFS: cephcsi.CephFS{
SubvolumeGroup: subvolGrpNameofCluster1,
},
},
}
s, err = updateCsiClusterConfig(s, subvolGrpNameofCluster1, &subVolCsiCluster1Config)
@@ -206,12 +250,13 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
assert.Equal(t, subvolGrpNameofCluster1, cc[1].CephFS.SubvolumeGroup, cc)
// add rados to same cluster
radosNsCsiCluster1Config := CsiClusterConfigEntry{
Namespace: clusterIDofCluster1,
Monitors: csiCluster1ConfigEntry.Monitors,
RadosNamespace: radosNSofCluster1,
RBD: &CsiRBDSpec{
RadosNamespace: radosNSofCluster1,
radosNsCsiCluster1Config := CSIClusterConfigEntry{
Namespace: clusterIDofCluster1,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: csiCluster1ConfigEntry.Monitors,
RBD: cephcsi.RBD{
RadosNamespace: radosNSofCluster1,
},
},
}
s, err = updateCsiClusterConfig(s, radosNSofCluster1, &radosNsCsiCluster1Config)
@@ -220,8 +265,6 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 3, len(cc), cc)
assert.Equal(t, 1, len(cc[2].Monitors))
// Now the configuration of new entries goes into RBD.RadosNamespace so it should be empty
assert.Empty(t, cc[2].RadosNamespace, cc)
assert.Equal(t, radosNSofCluster1, cc[2].RBD.RadosNamespace, cc)
// update mon IP's and check is it updating for all clusterID's
@@ -241,21 +284,29 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
radosNSofCluster2 := "rados-ns-2"
cluster2Mons := []string{"192.168.0.2:5000"}
csiCluster2ConfigEntry := CsiClusterConfigEntry{
csiCluster2ConfigEntry := CSIClusterConfigEntry{
Namespace: clusterIDofCluster2,
Monitors: cluster2Mons,
}
subVolCsiCluster2Config := CsiClusterConfigEntry{
Namespace: clusterIDofCluster2,
Monitors: cluster2Mons,
CephFS: &CsiCephFSSpec{
SubvolumeGroup: subvolGrpNameofCluster2,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: cluster2Mons,
},
}
radosNsCsiCluster2Config := CsiClusterConfigEntry{
Namespace: clusterIDofCluster2,
Monitors: cluster2Mons,
RadosNamespace: radosNSofCluster2,
subVolCsiCluster2Config := CSIClusterConfigEntry{
Namespace: clusterIDofCluster2,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: cluster2Mons,
CephFS: cephcsi.CephFS{
SubvolumeGroup: subvolGrpNameofCluster2,
},
},
}
radosNsCsiCluster2Config := CSIClusterConfigEntry{
Namespace: clusterIDofCluster2,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: cluster2Mons,
RBD: cephcsi.RBD{
RadosNamespace: radosNSofCluster2,
},
},
}
s, err = updateCsiClusterConfig(s, clusterIDofCluster2, &csiCluster2ConfigEntry)
assert.NoError(t, err)
@@ -323,9 +374,8 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
t.Run("test multus cluster", func(t *testing.T) {
s, err = updateCsiClusterConfig("[]", "rook-ceph-1", &csiClusterConfigEntryMultus)
assert.NoError(t, err)
assert.Equal(t, `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph-1","rbd":{"netNamespaceFilePath":"/var/run/netns/rook-ceph-1","radosNamespace":"rook-ceph-1"}}]`, s)
compareJSON(t, `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"rbd":{"netNamespaceFilePath":"/var/run/netns/rook-ceph-1","radosNamespace":"rook-ceph-1"},"namespace":"rook-ceph-1"}]`, s)
})
}
func contains(src, dest []string) bool {
@@ -408,21 +458,23 @@ func TestUpdateCSIDriverOptions(t *testing.T) {
{
name: "empty current config",
args: args{
clusterConfig: []CsiClusterConfigEntry{},
clusterConfig: []CSIClusterConfigEntry{},
clusterKey: "rook-ceph",
csiDriverOptions: &cephv1.CSIDriverSpec{},
},
want: []CsiClusterConfigEntry{},
want: []CSIClusterConfigEntry{},
wantErr: false,
},
{
name: "single matching current config",
args: args{
clusterConfig: []CsiClusterConfigEntry{
clusterConfig: []CSIClusterConfigEntry{
{
ClusterID: "rook-ceph",
Namespace: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
},
},
},
clusterKey: "rook-ceph",
@@ -437,18 +489,20 @@ func TestUpdateCSIDriverOptions(t *testing.T) {
},
},
},
want: []CsiClusterConfigEntry{
want: []CSIClusterConfigEntry{
{
ClusterID: "rook-ceph",
Namespace: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ReadAffinity: &cephv1.ReadAffinitySpec{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: &CsiCephFSSpec{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: cephcsi.CephFS{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
},
},
},
},
@@ -457,21 +511,27 @@ func TestUpdateCSIDriverOptions(t *testing.T) {
{
name: "multiple matching current config",
args: args{
clusterConfig: []CsiClusterConfigEntry{
clusterConfig: []CSIClusterConfigEntry{
{
ClusterID: "rook-ceph",
Namespace: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
},
},
{
ClusterID: "rook-ceph-2",
Namespace: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph-2",
Monitors: []string{"1.1.1.1"},
},
},
{
ClusterID: "rook-ceph-3",
Namespace: "rook-ceph-1",
Monitors: []string{"1.1.1.1"},
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph-3",
Monitors: []string{"1.1.1.1"},
},
},
},
clusterKey: "rook-ceph",
@@ -486,37 +546,43 @@ func TestUpdateCSIDriverOptions(t *testing.T) {
},
},
},
want: []CsiClusterConfigEntry{
want: []CSIClusterConfigEntry{
{
ClusterID: "rook-ceph",
Namespace: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ReadAffinity: &cephv1.ReadAffinitySpec{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: &CsiCephFSSpec{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: cephcsi.CephFS{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
},
},
},
{
ClusterID: "rook-ceph-2",
Namespace: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ReadAffinity: &cephv1.ReadAffinitySpec{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: &CsiCephFSSpec{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph-2",
Monitors: []string{"1.1.1.1"},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: cephcsi.CephFS{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
},
},
},
{
ClusterID: "rook-ceph-3",
Namespace: "rook-ceph-1",
Monitors: []string{"1.1.1.1"},
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph-3",
Monitors: []string{"1.1.1.1"},
},
},
},
wantErr: false,
pkg/operator/ceph/csi/spec.go
+15 -8
View File
@@ -36,6 +36,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/version"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
)
type Param struct {
@@ -865,15 +867,20 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp
}
}
clusterConfigEntry := &CsiClusterConfigEntry{
Monitors: MonEndpoints(c.clusterInfo.Monitors, c.cluster.Spec.RequireMsgr2()),
RBD: &CsiRBDSpec{},
CephFS: &CsiCephFSSpec{
FuseMountOptions: c.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions,
KernelMountOptions: c.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions,
clusterConfigEntry := &CSIClusterConfigEntry{
ClusterInfo: cephcsi.ClusterInfo{
Monitors: MonEndpoints(c.clusterInfo.Monitors, c.cluster.Spec.RequireMsgr2()),
RBD: cephcsi.RBD{},
CephFS: cephcsi.CephFS{
FuseMountOptions: c.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions,
KernelMountOptions: c.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions,
},
NFS: cephcsi.NFS{},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: c.clusterInfo.CSIDriverSpec.ReadAffinity.Enabled,
CrushLocationLabels: c.clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels,
},
},
NFS: &CsiNFSSpec{},
ReadAffinity: &c.clusterInfo.CSIDriverSpec.ReadAffinity,
}
netNamespaceFilePath := generateNetNamespaceFilePath(CSIParam.KubeletDirPath, driver.fullName, c.cluster.Namespace)
if driver.name == RBDDriverShortName {
pkg/operator/ceph/file/subvolumegroup/controller.go
+14 -7
View File
@@ -46,6 +46,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
)
const (
@@ -278,15 +280,20 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateClusterConfig(cephFilesyst
// Update CSI config map
// If the mon endpoints change, the mon health check go routine will take care of updating the
// config map, so no special care is needed in this controller
csiClusterConfigEntry := csi.CsiClusterConfigEntry{
csiClusterConfigEntry := csi.CSIClusterConfigEntry{
Namespace: r.clusterInfo.Namespace,
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
CephFS: &csi.CsiCephFSSpec{
SubvolumeGroup: getSubvolumeGroupName(cephFilesystemSubVolumeGroup),
KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions,
FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
CephFS: cephcsi.CephFS{
SubvolumeGroup: getSubvolumeGroupName(cephFilesystemSubVolumeGroup),
KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions,
FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions,
},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: r.clusterInfo.CSIDriverSpec.ReadAffinity.Enabled,
CrushLocationLabels: r.clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels,
},
},
ReadAffinity: &r.clusterInfo.CSIDriverSpec.ReadAffinity,
}
// If the cluster has Multus enabled we need to append the network namespace of the driver's
pkg/operator/ceph/pool/radosnamespace/controller.go
+16 -10
View File
@@ -44,6 +44,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
)
const (
@@ -264,18 +266,22 @@ func (r *ReconcileCephBlockPoolRadosNamespace) updateClusterConfig(cephBlockPool
// Update CSI config map
// If the mon endpoints change, the mon health check go routine will take care of updating the
// config map, so no special care is needed in this controller
csiClusterConfigEntry := csi.CsiClusterConfigEntry{
csiClusterConfigEntry := csi.CSIClusterConfigEntry{
Namespace: r.clusterInfo.Namespace,
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
RBD: &csi.CsiRBDSpec{
RadosNamespace: getRadosNamespaceName(cephBlockPoolRadosNamespace),
ClusterInfo: cephcsi.ClusterInfo{
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
RBD: cephcsi.RBD{
RadosNamespace: getRadosNamespaceName(cephBlockPoolRadosNamespace),
},
CephFS: cephcsi.CephFS{
KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions,
FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions,
},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: r.clusterInfo.CSIDriverSpec.ReadAffinity.Enabled,
CrushLocationLabels: r.clusterInfo.CSIDriverSpec.ReadAffinity.CrushLocationLabels,
},
},
RadosNamespace: getRadosNamespaceName(cephBlockPoolRadosNamespace),
CephFS: &csi.CsiCephFSSpec{
KernelMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.KernelMountOptions,
FuseMountOptions: r.clusterInfo.CSIDriverSpec.CephFS.FuseMountOptions,
},
ReadAffinity: &r.clusterInfo.CSIDriverSpec.ReadAffinity,
}
if cephCluster.Spec.Network.IsMultus() {