feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity
Files
d8428eeba94de17ee992c12dbc7cea7dd5f33dfe
my-rook-config/pkg/operator/ceph/csi/cluster_config_test.go
T
Blaine Gardner 9a58da323d csi: fix missing namespace bug in csi cluster config map 
Someone testing the Multus holder pod removal feature encountered an
issue where the migration process failed to lead to a system state where
PVCs could be created successfully.

The root cause was found to be a ceph csi config map wherein the primary
CephCluster entry was lacking a value for the "namespace" field.

I observed this once in my development on the holder pod removal
feature, but I was unable to reproduce and assumed it was my own error.
Since this has been seen in a user environment, it must be that the
error is a race condition, and I am unable to determine the exact source
of the bug.

I do not believe this bug would be present if the code that updates the
CSI configmap were properly idempotent, but it has many conditions based
on prior states, and I was unable to determine how to resolve this
underlying impelementation pattern issue.

Instead, I opted to separate the `clusterKey` parameter into two clear
parts:
1. `clusterID` for when `clusterKey` is used as an analogue for
   `clusterID`
2. `clusterNamespace` for when `clusterKey` is used as an analogue for
   `clusterNamespace`

I added unit tests to ensure that SaveClusterConfig() will be able to
detect when the namespace is currently missing, and using the new
`clusterNamespace` field, it should always know what value to use as
input when correcting the bug in already-installed systems.

I also verified that this update works when the function simultaneously
removes netNamespaceFilePath entries, and that those entries are removed
properly.

Finally, manual testing also verifies the change.

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
(cherry picked from commit 4ff881240f)
2024-05-03 18:10:30 +00:00

1066 lines
35 KiB
Go
Raw Blame History

/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/topology"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
func unmarshal(s string) ([]CSIClusterConfigEntry, error) {
var csiClusterConfigEntry []CSIClusterConfigEntry
if err := json.Unmarshal([]byte(s), &csiClusterConfigEntry); err != nil {
return csiClusterConfigEntry, err
}
return csiClusterConfigEntry, nil
}
func compareJSON(t *testing.T, exceptedJSON, actualJSON string) {
var (
err error
expected, actual []CSIClusterConfigEntry
)
expected, err = unmarshal(exceptedJSON)
if err != nil {
t.Error(err)
}
actual, err = unmarshal(exceptedJSON)
if err != nil {
t.Error(err)
}
assert.Equal(t, expected, actual)
}
func TestUpdateCsiClusterConfig(t *testing.T) {
csiClusterConfigEntry := CSIClusterConfigEntry{
Namespace: "rook-ceph-1",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"1.2.3.4:5000"},
},
}
csiClusterConfigEntryMultus := CSIClusterConfigEntry{
Namespace: "rook-ceph-1",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"1.2.3.4:5000"},
RBD: cephcsi.RBD{
NetNamespaceFilePath: "/var/run/netns/rook-ceph-1",
RadosNamespace: "rook-ceph-1",
},
},
}
csiClusterConfigEntryMountOptions := CSIClusterConfigEntry{
Namespace: "rook-ceph-1",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"1.2.3.4:5000"},
CephFS: cephcsi.CephFS{
KernelMountOptions: "ms_mode=crc",
FuseMountOptions: "debug",
},
},
}
csiClusterConfigEntry2 := CSIClusterConfigEntry{
Namespace: "rook-ceph-2",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"20.1.1.1:5000", "20.1.1.2:5000", "20.1.1.3:5000"},
},
}
csiClusterConfigEntry3 := CSIClusterConfigEntry{
Namespace: "rook-ceph-3",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"10.1.1.1:5000", "10.1.1.2:5000", "10.1.1.3:5000"},
CephFS: cephcsi.CephFS{
SubvolumeGroup: "my-group",
},
},
}
csiClusterConfigEntry4 := CSIClusterConfigEntry{
Namespace: "rook-ceph-4",
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"10.1.1.1:5000"},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: true,
CrushLocationLabels: strings.Split(topology.GetDefaultTopologyLabels(), ","),
},
},
}
var want, s string
var err error
t.Run("add a simple mons list", func(t *testing.T) {
s, err = updateCsiClusterConfig("[]", "rook-ceph-1", "rook-ceph-1", &csiClusterConfigEntry)
assert.NoError(t, err)
want = `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph-1"}]`
compareJSON(t, want, s)
})
t.Run("add a 2nd mon to the current cluster", func(t *testing.T) {
csiClusterConfigEntry.Monitors = append(csiClusterConfigEntry.Monitors, "10.11.12.13:5000")
s, err = updateCsiClusterConfig(s, "rook-ceph-1", "rook-ceph-1", &csiClusterConfigEntry)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 1, len(cc))
assert.Equal(t, "rook-ceph-1", cc[0].ClusterID)
assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
assert.Equal(t, 2, len(cc[0].Monitors))
})
t.Run("add mount options to the current cluster", func(t *testing.T) {
configWithMountOptions, err := updateCsiClusterConfig(s, "rook-ceph-1", "rook-ceph-1", &csiClusterConfigEntryMountOptions)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(configWithMountOptions)
assert.NoError(t, err)
assert.Equal(t, 1, len(cc))
assert.Equal(t, "rook-ceph-1", cc[0].ClusterID)
assert.Equal(t, csiClusterConfigEntryMountOptions.CephFS.KernelMountOptions, cc[0].CephFS.KernelMountOptions)
assert.Equal(t, csiClusterConfigEntryMountOptions.CephFS.FuseMountOptions, cc[0].CephFS.FuseMountOptions)
})
t.Run("add a 2nd cluster with 3 mons", func(t *testing.T) {
s, err = updateCsiClusterConfig(s, "beta", "beta", &csiClusterConfigEntry2)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 2, len(cc))
assert.Equal(t, "rook-ceph-1", cc[0].ClusterID)
assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
// check 1st cluster contains any of the 3 mons from 2nd cluster
assert.NotContains(t, cc[0].Monitors, "20.1.1.1:5000")
assert.Equal(t, 2, len(cc[0].Monitors))
assert.Equal(t, "beta", cc[1].ClusterID)
assert.Contains(t, cc[1].Monitors, "20.1.1.1:5000")
assert.Contains(t, cc[1].Monitors, "20.1.1.2:5000")
assert.Contains(t, cc[1].Monitors, "20.1.1.3:5000")
// check 2nd cluster contains any of the mons from 1st cluster
assert.NotContains(t, cc[1].Monitors, "10.11.12.13:5000")
assert.Equal(t, 3, len(cc[1].Monitors))
})
t.Run("remove a mon from the 2nd cluster", func(t *testing.T) {
i := 2
// Remove last element of the slice
csiClusterConfigEntry2.Monitors = append(csiClusterConfigEntry2.Monitors[:i], csiClusterConfigEntry2.Monitors[i+1:]...)
s, err = updateCsiClusterConfig(s, "beta", "beta", &csiClusterConfigEntry2)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 2, len(cc))
assert.Equal(t, "rook-ceph-1", cc[0].ClusterID)
assert.Contains(t, cc[0].Monitors, "1.2.3.4:5000")
assert.Contains(t, cc[0].Monitors, "10.11.12.13:5000")
assert.Equal(t, 2, len(cc[0].Monitors))
assert.Equal(t, "beta", cc[1].ClusterID)
assert.Contains(t, cc[1].Monitors, "20.1.1.1:5000")
assert.Contains(t, cc[1].Monitors, "20.1.1.2:5000")
assert.Equal(t, 2, len(cc[1].Monitors))
})
t.Run("add a 3rd cluster with subvolumegroup", func(t *testing.T) {
s, err = updateCsiClusterConfig(s, "baba", "baba", &csiClusterConfigEntry3)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 3, len(cc))
assert.Equal(t, "rook-ceph-1", cc[0].ClusterID)
assert.Equal(t, 2, len(cc[0].Monitors))
assert.Equal(t, "beta", cc[1].ClusterID)
assert.Equal(t, "baba", cc[2].ClusterID)
assert.Equal(t, "10.1.1.1:5000", cc[2].Monitors[0])
assert.Equal(t, 3, len(cc[2].Monitors))
assert.Equal(t, "my-group", cc[2].CephFS.SubvolumeGroup)
})
t.Run("update mount options in presence of subvolumegroup", func(t *testing.T) {
sMntOptionUpdate, err := updateCsiClusterConfig(s, "baba", "baba", &csiClusterConfigEntryMountOptions)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(sMntOptionUpdate)
assert.NoError(t, err)
assert.Equal(t, 3, len(cc))
assert.Equal(t, "baba", cc[2].ClusterID)
assert.Equal(t, "my-group", cc[2].CephFS.SubvolumeGroup)
assert.Equal(t, csiClusterConfigEntryMountOptions.CephFS.KernelMountOptions, cc[2].CephFS.KernelMountOptions)
assert.Equal(t, csiClusterConfigEntryMountOptions.CephFS.FuseMountOptions, cc[2].CephFS.FuseMountOptions)
})
t.Run("add a 4th mon to the 3rd cluster and subvolumegroup is preserved", func(t *testing.T) {
csiClusterConfigEntry3.Monitors = append(csiClusterConfigEntry3.Monitors, "10.11.12.13:5000")
s, err = updateCsiClusterConfig(s, "baba", "baba", &csiClusterConfigEntry3)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 3, len(cc))
assert.Equal(t, 4, len(cc[2].Monitors))
assert.Equal(t, "my-group", cc[2].CephFS.SubvolumeGroup)
})
t.Run("remove subvolumegroup", func(t *testing.T) {
csiClusterConfigEntry3.CephFS.SubvolumeGroup = ""
s, err = updateCsiClusterConfig(s, "baba", "baba", nil)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 2, len(cc)) // we only have 2 clusters now
})
t.Run("add subvolumegroup and mons after", func(t *testing.T) {
csiClusterConfigEntry4 := CSIClusterConfigEntry{
ClusterInfo: cephcsi.ClusterInfo{
CephFS: cephcsi.CephFS{
SubvolumeGroup: "my-group2",
},
},
}
s, err = updateCsiClusterConfig(s, "quatre", "quatre", &csiClusterConfigEntry4)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 3, len(cc), cc)
assert.Equal(t, 0, len(cc[2].Monitors))
assert.Equal(t, "my-group2", cc[2].CephFS.SubvolumeGroup, cc)
csiClusterConfigEntry4.Monitors = []string{"10.1.1.1:5000", "10.1.1.2:5000", "10.1.1.3:5000"}
s, err = updateCsiClusterConfig(s, "quatre", "quatre", &csiClusterConfigEntry4)
assert.NoError(t, err)
cc, err = parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, "my-group2", cc[2].CephFS.SubvolumeGroup)
assert.Equal(t, 3, len(cc[2].Monitors))
})
t.Run("does it return error on garbage input?", func(t *testing.T) {
_, err = updateCsiClusterConfig("qqq", "beta", "beta", &csiClusterConfigEntry2)
assert.Error(t, err)
})
t.Run("test mon IP's update across all clusterID's belong to same cluster", func(t *testing.T) {
clusterIDofCluster1 := "rook-ceph"
subvolGrpNameofCluster1 := "subvol-group"
radosNSofCluster1 := "rados-ns"
csiCluster1ConfigEntry := CSIClusterConfigEntry{
Namespace: clusterIDofCluster1,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"1.2.3.4:5000"},
},
}
s, err := updateCsiClusterConfig("[]", clusterIDofCluster1, clusterIDofCluster1, &csiCluster1ConfigEntry)
assert.NoError(t, err)
want = `[{"clusterID":"rook-ceph","monitors":["1.2.3.4:5000"],"namespace":"rook-ceph"}]`
compareJSON(t, want, s)
// add subvolumegroup to same cluster
subVolCsiCluster1Config := CSIClusterConfigEntry{
Namespace: clusterIDofCluster1,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: csiCluster1ConfigEntry.Monitors,
CephFS: cephcsi.CephFS{
SubvolumeGroup: subvolGrpNameofCluster1,
},
},
}
s, err = updateCsiClusterConfig(s, subvolGrpNameofCluster1, clusterIDofCluster1, &subVolCsiCluster1Config)
assert.NoError(t, err)
cc, err := parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 2, len(cc), cc)
assert.Equal(t, 1, len(cc[1].Monitors))
assert.Equal(t, subvolGrpNameofCluster1, cc[1].CephFS.SubvolumeGroup, cc)
// add rados to same cluster
radosNsCsiCluster1Config := CSIClusterConfigEntry{
Namespace: clusterIDofCluster1,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: csiCluster1ConfigEntry.Monitors,
RBD: cephcsi.RBD{
RadosNamespace: radosNSofCluster1,
},
},
}
s, err = updateCsiClusterConfig(s, radosNSofCluster1, clusterIDofCluster1, &radosNsCsiCluster1Config)
assert.NoError(t, err)
cc, err = parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 3, len(cc), cc)
assert.Equal(t, 1, len(cc[2].Monitors))
assert.Equal(t, radosNSofCluster1, cc[2].RBD.RadosNamespace, cc)
// update mon IP's and check is it updating for all clusterID's
csiCluster1ConfigEntry.Monitors = append(csiCluster1ConfigEntry.Monitors, "1.2.3.10:5000")
s, err = updateCsiClusterConfig(s, clusterIDofCluster1, clusterIDofCluster1, &csiCluster1ConfigEntry)
assert.NoError(t, err)
cc, err = parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 3, len(cc), cc)
// check mon entries are updated in all configmap
for _, i := range cc {
assert.Equal(t, 2, len(i.Monitors))
}
clusterIDofCluster2 := "rook-ceph-2"
subvolGrpNameofCluster2 := "subvol-group-2"
radosNSofCluster2 := "rados-ns-2"
cluster2Mons := []string{"192.168.0.2:5000"}
csiCluster2ConfigEntry := CSIClusterConfigEntry{
Namespace: clusterIDofCluster2,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: cluster2Mons,
},
}
subVolCsiCluster2Config := CSIClusterConfigEntry{
Namespace: clusterIDofCluster2,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: cluster2Mons,
CephFS: cephcsi.CephFS{
SubvolumeGroup: subvolGrpNameofCluster2,
},
},
}
radosNsCsiCluster2Config := CSIClusterConfigEntry{
Namespace: clusterIDofCluster2,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: cluster2Mons,
RBD: cephcsi.RBD{
RadosNamespace: radosNSofCluster2,
},
},
}
s, err = updateCsiClusterConfig(s, clusterIDofCluster2, clusterIDofCluster2, &csiCluster2ConfigEntry)
assert.NoError(t, err)
s, err = updateCsiClusterConfig(s, subvolGrpNameofCluster2, clusterIDofCluster2, &subVolCsiCluster2Config)
assert.NoError(t, err)
s, err = updateCsiClusterConfig(s, radosNSofCluster2, clusterIDofCluster2, &radosNsCsiCluster2Config)
assert.NoError(t, err)
cc, err = parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 6, len(cc), cc)
// check mon entries of 1st cluster amd mon overlapping with 2nd cluster
for i := 0; i < 3; i++ {
assert.Equal(t, 2, len(cc[i].Monitors))
assert.False(t, contains(cc[i].Monitors, cluster2Mons))
}
// check mon entries of 2nd cluster
for i := 3; i < 6; i++ {
assert.Equal(t, 1, len(cc[i].Monitors))
assert.False(t, contains(cc[i].Monitors, csiCluster1ConfigEntry.Monitors))
}
// update mon on 2nd cluster and check is it updating for all clusterID's
// of 2nd cluster
csiCluster2ConfigEntry.Monitors = append(csiCluster2ConfigEntry.Monitors, "192.168.0.3:5000")
s, err = updateCsiClusterConfig(s, clusterIDofCluster2, clusterIDofCluster2, &csiCluster2ConfigEntry)
assert.NoError(t, err)
cc, err = parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 6, len(cc), cc)
// check mon entries are updated
for _, i := range cc {
assert.Equal(t, 2, len(i.Monitors))
}
// check for overlapping
// check mon entries of 1st cluster amd mon overlapping with 2nd cluster
for i := 0; i < 3; i++ {
assert.False(t, contains(cc[i].Monitors, cluster2Mons))
}
// check mon entries of 2nd cluster
for i := 3; i < 6; i++ {
assert.False(t, contains(cc[i].Monitors, csiCluster1ConfigEntry.Monitors))
}
// Remove one mon from 2nd cluster and check is it updating for all
// clusterID's of 2nd cluster
i := 1
csiCluster2ConfigEntry.Monitors = append(csiCluster2ConfigEntry.Monitors[:i], csiCluster2ConfigEntry.Monitors[i+1:]...)
s, err = updateCsiClusterConfig(s, clusterIDofCluster2, clusterIDofCluster2, &csiCluster2ConfigEntry)
assert.NoError(t, err)
cc, err = parseCsiClusterConfig(s)
assert.NoError(t, err)
assert.Equal(t, 6, len(cc), cc)
// check for overlapping
// check mon entries of 1st cluster amd mon overlapping with 2nd cluster
for i := 0; i < 3; i++ {
assert.False(t, contains(cc[i].Monitors, cluster2Mons))
assert.Equal(t, 2, len(cc[i].Monitors))
}
// check mon entries of 2nd cluster
for i := 3; i < 6; i++ {
assert.False(t, contains(cc[i].Monitors, csiCluster1ConfigEntry.Monitors))
assert.Equal(t, 1, len(cc[i].Monitors))
}
})
t.Run("test multus cluster", func(t *testing.T) {
s, err = updateCsiClusterConfig("[]", "rook-ceph-1", "rook-ceph-1", &csiClusterConfigEntryMultus)
assert.NoError(t, err)
compareJSON(t, `[{"clusterID":"rook-ceph-1","monitors":["1.2.3.4:5000"],"rbd":{"netNamespaceFilePath":"/var/run/netns/rook-ceph-1","radosNamespace":"rook-ceph-1"},"namespace":"rook-ceph-1"}]`, s)
})
t.Run("test crush location labels are set", func(t *testing.T) {
s, err = updateCsiClusterConfig("[]", "rook-ceph-4", "rook-ceph-4", &csiClusterConfigEntry4)
assert.NoError(t, err)
compareJSON(t, `[{"clusterID":"rook-ceph-4","monitors":["10.1.1.1:5000"],"readAffinity": {"enabled": true, "crushLocationLabels":["kubernetes.io/hostname",
"topology.kubernetes.io/region","topology.kubernetes.io/zone","topology.rook.io/chassis","topology.rook.io/rack","topology.rook.io/row","topology.rook.io/pdu",
"topology.rook.io/pod","topology.rook.io/room","topology.rook.io/datacenter"]},"namespace":"rook-ceph-4"}]`, s)
})
t.Run("test empty namespace correction", func(t *testing.T) {
currentConfigFormatString := `[` +
`{"clusterID":"cluster-id-and-namespace","monitors":["172.30.100.1:3300"],"cephFS":{"netNamespaceFilePath":"","subvolumeGroup":"","kernelMountOptions":"","fuseMountOptions":""},"rbd":{"netNamespaceFilePath":"","radosNamespace":""},"nfs":{"netNamespaceFilePath":""},"readAffinity":{"enabled":false,"crushLocationLabels":null},"namespace":"%s"},` +
`{"clusterID":"5bb69c306a7d011c3e91c3cec112fb7a","monitors":["172.30.100.1:3300"],"cephFS":{"netNamespaceFilePath":"","subvolumeGroup":"csi","kernelMountOptions":"","fuseMountOptions":""},"rbd":{"netNamespaceFilePath":"","radosNamespace":""},"nfs":{"netNamespaceFilePath":""},"readAffinity":{"enabled":false,"crushLocationLabels":null},"namespace":"cluster-id-and-namespace"}` +
`]`
idAndNs := "cluster-id-and-namespace"
csiConfigEntry := &CSIClusterConfigEntry{
Namespace: idAndNs,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"172.30.100.1:3300"},
},
}
currentConfigWithoutNamespace := fmt.Sprintf(currentConfigFormatString, "")
out, err := updateCsiClusterConfig(currentConfigWithoutNamespace, idAndNs, idAndNs, csiConfigEntry)
assert.NoError(t, err)
expectedOutput := fmt.Sprintf(currentConfigFormatString, idAndNs)
assert.Equal(t, expectedOutput, out)
})
t.Run("test empty namespace correction and clear netNamespaceFilePath", func(t *testing.T) {
holderEnabled = false
currentConfigFormatString := `[` +
`{"clusterID":"cluster-id-and-namespace","monitors":["172.30.100.1:3300"],"cephFS":{"netNamespaceFilePath":"%s","subvolumeGroup":"","kernelMountOptions":"","fuseMountOptions":""},"rbd":{"netNamespaceFilePath":"%s","radosNamespace":""},"nfs":{"netNamespaceFilePath":""},"readAffinity":{"enabled":false,"crushLocationLabels":null},"namespace":"%s"},` +
`{"clusterID":"5bb69c306a7d011c3e91c3cec112fb7a","monitors":["172.30.100.1:3300"],"cephFS":{"netNamespaceFilePath":"","subvolumeGroup":"csi","kernelMountOptions":"","fuseMountOptions":""},"rbd":{"netNamespaceFilePath":"","radosNamespace":""},"nfs":{"netNamespaceFilePath":""},"readAffinity":{"enabled":false,"crushLocationLabels":null},"namespace":"cluster-id-and-namespace"}` +
`]`
cephFsNetNsFilePath := "/var/lib/kubelet/plugins/cluster-id-and-namespace.cephfs.csi.ceph.com/cluster-id-and-namespace.net.ns"
rbdNetNsFilePath := "/var/lib/kubelet/plugins/cluster-id-and-namespace.rbd.csi.ceph.com/cluster-id-and-namespace.net.ns"
idAndNs := "cluster-id-and-namespace"
csiConfigEntry := &CSIClusterConfigEntry{
Namespace: idAndNs,
ClusterInfo: cephcsi.ClusterInfo{
Monitors: []string{"172.30.100.1:3300"},
},
}
currentConfigWithoutNamespace := fmt.Sprintf(currentConfigFormatString, cephFsNetNsFilePath, rbdNetNsFilePath, "")
out, err := updateCsiClusterConfig(currentConfigWithoutNamespace, idAndNs, idAndNs, csiConfigEntry)
assert.NoError(t, err)
expectedOutput := fmt.Sprintf(currentConfigFormatString, "", "", idAndNs)
assert.Equal(t, expectedOutput, out)
})
}
func contains(src, dest []string) bool {
for _, s := range src {
for _, d := range dest {
if s == d {
return true
}
}
}
return false
}
func TestMonEndpoints(t *testing.T) {
monInfo := map[string]*cephclient.MonInfo{
"a": {Name: "a", Endpoint: "1.2.3.4:6789"},
"b": {Name: "b", Endpoint: "1.2.3.5:6789"},
"c": {Name: "c", Endpoint: "1.2.3.6:6789"},
}
t.Run("msgrv1 when not require msgr2", func(t *testing.T) {
endpoints := MonEndpoints(monInfo, false)
assert.Equal(t, 3, len(endpoints))
verifyEndpointPort(t, endpoints, "6789")
})
t.Run("convert to msgr2", func(t *testing.T) {
endpoints := MonEndpoints(monInfo, true)
assert.Equal(t, 3, len(endpoints))
verifyEndpointPort(t, endpoints, "3300")
})
t.Run("remains msgr2", func(t *testing.T) {
monInfo := map[string]*cephclient.MonInfo{
"a": {Name: "a", Endpoint: "1.2.3.4:3300"},
"b": {Name: "b", Endpoint: "1.2.3.5:3300"},
"c": {Name: "c", Endpoint: "1.2.3.6:3300"},
}
endpoints := MonEndpoints(monInfo, false)
assert.Equal(t, 3, len(endpoints))
verifyEndpointPort(t, endpoints, "3300")
})
t.Run("ipv6 endpoint conversion", func(t *testing.T) {
monInfo := map[string]*cephclient.MonInfo{
"a": {Name: "a", Endpoint: "[fd07:aaaa:bbbb:cccc::11]:6789"},
"b": {Name: "a", Endpoint: "[1234:6789:bbbb:cccc::11]:6789"},
}
endpoints := MonEndpoints(monInfo, true)
assert.Equal(t, 2, len(endpoints))
verifyEndpointPort(t, endpoints, "3300")
for _, endpoint := range endpoints {
// Verify that the v1 port inside the ipv6 address will not be replaced
if strings.HasPrefix(endpoint, "[1234") {
assert.True(t, strings.HasPrefix(endpoint, "[1234:6789"))
}
}
})
}
func verifyEndpointPort(t *testing.T, endpoints []string, expectedPort string) {
for _, endpoint := range endpoints {
assert.True(t, strings.HasSuffix(endpoint, expectedPort))
}
}
func TestUpdateCSIDriverOptions(t *testing.T) {
type args struct {
clusterConfig csiClusterConfig
clusterKey string
csiDriverOptions *cephv1.CSIDriverSpec
}
holderEnabled = true
tests := []struct {
name string
args args
want csiClusterConfig
wantErr bool
}{
{
name: "empty current config",
args: args{
clusterConfig: []CSIClusterConfigEntry{},
clusterKey: "rook-ceph",
csiDriverOptions: &cephv1.CSIDriverSpec{},
},
want: []CSIClusterConfigEntry{},
wantErr: false,
},
{
name: "single matching current config",
args: args{
clusterConfig: []CSIClusterConfigEntry{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
CephFS: cephcsi.CephFS{
NetNamespaceFilePath: "netnamespacefilepath",
},
},
},
},
clusterKey: "rook-ceph",
csiDriverOptions: &cephv1.CSIDriverSpec{
ReadAffinity: cephv1.ReadAffinitySpec{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: cephv1.CSICephFSSpec{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
},
},
},
want: []CSIClusterConfigEntry{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: cephcsi.CephFS{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
NetNamespaceFilePath: "netnamespacefilepath",
},
},
},
},
wantErr: false,
},
{
name: "multiple matching current config",
args: args{
clusterConfig: []CSIClusterConfigEntry{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
},
},
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph-2",
Monitors: []string{"1.1.1.1"},
},
},
{
Namespace: "rook-ceph-1",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph-3",
Monitors: []string{"1.1.1.1"},
},
},
},
clusterKey: "rook-ceph",
csiDriverOptions: &cephv1.CSIDriverSpec{
ReadAffinity: cephv1.ReadAffinitySpec{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: cephv1.CSICephFSSpec{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
},
},
},
want: []CSIClusterConfigEntry{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: cephcsi.CephFS{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
},
},
},
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph-2",
Monitors: []string{"1.1.1.1"},
ReadAffinity: cephcsi.ReadAffinity{
Enabled: true,
CrushLocationLabels: []string{"topology.rook.io/rack"},
},
CephFS: cephcsi.CephFS{
KernelMountOptions: "rw,noatime",
FuseMountOptions: "debug",
},
},
},
{
Namespace: "rook-ceph-1",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph-3",
Monitors: []string{"1.1.1.1"},
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dataString, err := formatCsiClusterConfig(tt.args.clusterConfig)
assert.NoError(t, err)
got, err := updateCSIDriverOptions(dataString, tt.args.clusterKey, tt.args.csiDriverOptions)
if (err != nil) != tt.wantErr {
t.Errorf("updateCSIDriverOptions() error = %v, wantErr %v", err, tt.wantErr)
return
}
expectedString, err := formatCsiClusterConfig(tt.want)
assert.NoError(t, err)
assert.Equal(t, expectedString, got)
})
}
}
func TestUpdateNetNamespaceFilePath(t *testing.T) {
type args struct {
clusterConfig csiClusterConfig
clusterKey string
holderEnabled bool
}
tests := []struct {
name string
args args
want csiClusterConfig
}{
{
name: "empty",
args: args{
clusterKey: "rook-ceph",
holderEnabled: false,
clusterConfig: []CSIClusterConfigEntry{},
},
want: []CSIClusterConfigEntry{},
},
{
name: "holder enabled",
args: args{
clusterKey: "rook-ceph",
holderEnabled: true,
clusterConfig: []CSIClusterConfigEntry{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
CephFS: cephcsi.CephFS{
NetNamespaceFilePath: "cephfs.net.ns",
},
RBD: cephcsi.RBD{
NetNamespaceFilePath: "rbd.net.ns",
},
NFS: cephcsi.NFS{
NetNamespaceFilePath: "nfs.net.ns",
},
},
},
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "cluster-1",
Monitors: []string{"1.1.1.1"},
CephFS: cephcsi.CephFS{
SubvolumeGroup: "csi",
},
},
},
},
},
want: []CSIClusterConfigEntry{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
CephFS: cephcsi.CephFS{
NetNamespaceFilePath: "cephfs.net.ns",
},
RBD: cephcsi.RBD{
NetNamespaceFilePath: "rbd.net.ns",
},
NFS: cephcsi.NFS{
NetNamespaceFilePath: "nfs.net.ns",
},
},
},
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "cluster-1",
Monitors: []string{"1.1.1.1"},
CephFS: cephcsi.CephFS{
SubvolumeGroup: "csi",
NetNamespaceFilePath: "cephfs.net.ns",
},
RBD: cephcsi.RBD{
NetNamespaceFilePath: "rbd.net.ns",
},
NFS: cephcsi.NFS{
NetNamespaceFilePath: "nfs.net.ns",
},
},
},
},
},
{
name: "holder disabled",
args: args{
clusterKey: "rook-ceph",
holderEnabled: false,
clusterConfig: []CSIClusterConfigEntry{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
CephFS: cephcsi.CephFS{
NetNamespaceFilePath: "cephfs.net.ns",
},
RBD: cephcsi.RBD{
NetNamespaceFilePath: "rbd.net.ns",
},
NFS: cephcsi.NFS{
NetNamespaceFilePath: "nfs.net.ns",
},
},
},
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "cluster-1",
Monitors: []string{"1.1.1.1"},
RBD: cephcsi.RBD{
RadosNamespace: "group-1",
},
},
},
},
},
want: []CSIClusterConfigEntry{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
Monitors: []string{"1.1.1.1"},
CephFS: cephcsi.CephFS{
NetNamespaceFilePath: "",
},
RBD: cephcsi.RBD{
NetNamespaceFilePath: "",
},
NFS: cephcsi.NFS{
NetNamespaceFilePath: "",
},
},
},
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "cluster-1",
Monitors: []string{"1.1.1.1"},
CephFS: cephcsi.CephFS{
NetNamespaceFilePath: "",
},
RBD: cephcsi.RBD{
NetNamespaceFilePath: "",
RadosNamespace: "group-1",
},
NFS: cephcsi.NFS{
NetNamespaceFilePath: "",
},
},
},
},
},
}
cephfsNsFilePath := "cephfs.net.ns"
rbdNsFilePath := "rbd.net.ns"
nfsNsFilePath := "nfs.net.ns"
csiConfigMap := csiClusterConfig{
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
CephFS: cephcsi.CephFS{
NetNamespaceFilePath: cephfsNsFilePath,
},
RBD: cephcsi.RBD{
NetNamespaceFilePath: rbdNsFilePath,
},
NFS: cephcsi.NFS{
NetNamespaceFilePath: nfsNsFilePath,
},
},
},
{
Namespace: "rook-ceph",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "svg",
},
},
{
Namespace: "default",
ClusterInfo: cephcsi.ClusterInfo{
ClusterID: "rook-ceph",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
holderEnabled = tt.args.holderEnabled
updateNetNamespaceFilePath("rook-ceph", tt.args.clusterConfig)
assert.True(t, reflect.DeepEqual(tt.args.clusterConfig, tt.want))
})
}
t.Run("Holder enabled and disabled later", func(t *testing.T) {
holderEnabled = true
updateNetNamespaceFilePath("rook-ceph", csiConfigMap)
for _, c := range csiConfigMap {
if c.Namespace == "rook-ceph" {
assert.Equal(t, cephfsNsFilePath, c.CephFS.NetNamespaceFilePath)
assert.Equal(t, rbdNsFilePath, c.RBD.NetNamespaceFilePath)
assert.Equal(t, nfsNsFilePath, c.NFS.NetNamespaceFilePath)
}
}
holderEnabled = false
updateNetNamespaceFilePath("rook-ceph", csiConfigMap)
for _, c := range csiConfigMap {
if c.Namespace == "rook-ceph" {
assert.Equal(t, "", c.CephFS.NetNamespaceFilePath)
assert.Equal(t, "", c.RBD.NetNamespaceFilePath)
assert.Equal(t, "", c.NFS.NetNamespaceFilePath)
}
}
})
}
func Test_updateCsiConfigMapOwnerRefs(t *testing.T) {
ctx := context.TODO()
ns := "test-ns"
ownerController := true
blockOwnerDel := true
opDeployRef := metav1.OwnerReference{
APIVersion: "v1",
Kind: "Deployment",
Name: "rook-ceph-operator",
UID: "e55604f2-710c-4353-9a3e-9d23ea2d6eb9", // random uuid
Controller: &ownerController,
BlockOwnerDeletion: &blockOwnerDel,
}
expectedOwnerInfo := k8sutil.NewOwnerInfoWithOwnerRef(&opDeployRef, ns)
minimalCsiConfigMap := func() *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: ConfigName,
Namespace: ns,
OwnerReferences: []metav1.OwnerReference{},
},
}
}
t.Run("no configmap", func(t *testing.T) {
clientset := test.New(t, 1)
err := updateCsiConfigMapOwnerRefs(ctx, ns, clientset, expectedOwnerInfo)
assert.ErrorContains(t, err, "failed to fetch csi config map")
assert.ErrorContains(t, err, "which already exists")
})
assertOwner := func(t *testing.T, clientset kubernetes.Interface) {
t.Helper()
cm, err := clientset.CoreV1().ConfigMaps(ns).Get(ctx, ConfigName, metav1.GetOptions{})
assert.NoError(t, err)
assert.Len(t, cm.GetOwnerReferences(), 1)
cmOwner := cm.GetOwnerReferences()[0]
assert.Equal(t, "v1", cmOwner.APIVersion)
assert.Equal(t, "Deployment", cmOwner.Kind)
assert.Equal(t, "rook-ceph-operator", cmOwner.Name)
assert.Equal(t, "e55604f2-710c-4353-9a3e-9d23ea2d6eb9", string(cmOwner.UID))
assert.True(t, *cmOwner.Controller)
assert.True(t, *cmOwner.BlockOwnerDeletion)
}
t.Run("no existing owner ref", func(t *testing.T) {
clientset := test.New(t, 1)
cm := minimalCsiConfigMap()
_, err := clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{})
assert.NoError(t, err)
err = updateCsiConfigMapOwnerRefs(ctx, ns, clientset, expectedOwnerInfo)
assert.NoError(t, err)
assertOwner(t, clientset)
})
t.Run("correct existing owner ref", func(t *testing.T) {
clientset := test.New(t, 1)
cm := minimalCsiConfigMap()
cm.OwnerReferences = []metav1.OwnerReference{
*opDeployRef.DeepCopy(), // correct ref
}
_, err := clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{})
assert.NoError(t, err)
err = updateCsiConfigMapOwnerRefs(ctx, ns, clientset, expectedOwnerInfo)
assert.NoError(t, err)
assertOwner(t, clientset)
})
t.Run("single incorrect existing owner ref", func(t *testing.T) {
clientset := test.New(t, 1)
cm := minimalCsiConfigMap()
ownerController := true
blockOwnerDel := true
cm.OwnerReferences = []metav1.OwnerReference{
{
APIVersion: "ceph.rook.io/v1",
Kind: "CephCluster",
Name: "my-cluster",
UID: "a77777a7-777a-7777-7a7a-7a77aa7a7aa7", // random uuid
Controller: &ownerController,
BlockOwnerDeletion: &blockOwnerDel,
},
}
_, err := clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{})
assert.NoError(t, err)
err = updateCsiConfigMapOwnerRefs(ctx, ns, clientset, expectedOwnerInfo)
assert.NoError(t, err)
assertOwner(t, clientset)
})
t.Run("multiple existing owner refs, one correct", func(t *testing.T) {
clientset := test.New(t, 1)
cm := minimalCsiConfigMap()
ownerController := true
blockOwnerDel := true
cm.OwnerReferences = []metav1.OwnerReference{
*opDeployRef.DeepCopy(), // correct ref
{
APIVersion: "ceph.rook.io/v1",
Kind: "CephCluster",
Name: "my-cluster",
UID: "a77777a7-777a-7777-7a7a-7a77aa7a7aa7", // random uuid
Controller: &ownerController,
BlockOwnerDeletion: &blockOwnerDel,
},
}
_, err := clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{})
assert.NoError(t, err)
err = updateCsiConfigMapOwnerRefs(ctx, ns, clientset, expectedOwnerInfo)
assert.NoError(t, err)
assertOwner(t, clientset)
})
}
View Git Blame Copy Permalink