feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity
Files
a2e0b0c46b812d2d4aa27eb02bdfd75e4efe701e
my-rook-config/pkg/operator/ceph/controller/network_test.go
T
Blaine Gardner 3c43268d0a multus: detect network CIDRs via canary 
Change how Rook detects network CIDRs for Multus networks. The IPAM
configuration is only defined as an arbitrary string JSON blob with a
"type" field and nothing more. Rook's detection of CIDRs for whereabouts
had already grown out of date since the initial implementation.
Additionally, Rook did not support DHCP IPAM, which is a reasonable
choice for users. And more, Rook did not support CNI plugin chaining,
which further complicates NADs. Based on the CNI spec, network chaning
can result in any changes to network CIDRs from the first-given plugin.

All these problems make it more and more difficult for Rook to support
Multus by inspecting the NAD itself to predict network CIDRs. Instead,
it is better for Rook to treat the CNI process as a black box. To
preserve legacy functionality of auto-detecting networks and to make
that as robust as possible, change to a canary-style architecture like
that used for Ceph mons, from which Rook will detect the network CIDRs
if possible.

Also allow users to specify overrides for CIDR ranges. This allows Rook
to still support esoteric and unexpected NAD or network configurations
where a CIDR range is not detectable or where the range detected would
be incomplete. Because it may be impossible for Rook to understand the
network CIDRs wholistically while residing only on a portion of the
network, this feature should have been present from Multus's inception.

Improving CIDR auto-detection and allowing users to specify overrides
for auto-detected CIDRs rounds out Rook's Multus support for CephCluster
(core/RADOS) installations. No further architectural changes should be
needed for CephClusters as regards application of public/cluster network
CIDRs for Multus networks.

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
2023-09-07 10:12:55 -06:00

831 lines
34 KiB
Go
Raw Blame History

/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"context"
"fmt"
"strings"
"testing"
"time"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/k8sutil/cmdreporter"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
batch "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
type mockCmdReporter struct {
mock.Mock
job *batch.Job
}
func (m *mockCmdReporter) Job() *batch.Job {
m.Called()
return m.job
}
func (m *mockCmdReporter) Run(ctx context.Context, timeout time.Duration) (stdout, stderr string, retcode int, retErr error) {
args := m.Called(ctx, timeout)
return args.String(0), args.String(1), args.Int(2), args.Error(3)
}
// returns a newCmdReporter function that returns the given mockCmdReporter and error
func mockNewCmdReporter(m *mockCmdReporter, err error) func(clientset kubernetes.Interface, ownerInfo *k8sutil.OwnerInfo, appName string, jobName string, jobNamespace string, cmd []string, args []string, rookImage string, runImage string, imagePullPolicy v1.PullPolicy) (cmdreporter.CmdReporterInterface, error) {
return func(clientset kubernetes.Interface, ownerInfo *k8sutil.OwnerInfo, appName, jobName, jobNamespace string, cmd, args []string, rookImage, runImage string, imagePullPolicy v1.PullPolicy) (cmdreporter.CmdReporterInterface, error) {
job, err := cmdreporter.MockCmdReporterJob(clientset, ownerInfo, appName, jobName, jobNamespace, cmd, args, rookImage, runImage, imagePullPolicy)
if err != nil {
// okay to panic here because this is a unit test setup failure, not part of code testing
panic(fmt.Sprintf("error setting up mock CmdReporter job: %v", err))
}
m.job = job
return m, err
}
}
func newTestConfigsWithNetworkSpec(n cephv1.NetworkSpec) (*clusterd.Context, *cephv1.ClusterSpec, *client.ClusterInfo) {
clusterdCtx := &clusterd.Context{}
clusterSpec := &cephv1.ClusterSpec{
Network: n,
}
clusterInfo := &client.ClusterInfo{
Namespace: "ns",
NetworkSpec: clusterSpec.Network,
OwnerInfo: k8sutil.NewOwnerInfoWithOwnerRef(&metav1.OwnerReference{}, "ns"),
}
return clusterdCtx, clusterSpec, clusterInfo
}
func Test_discoverAddressRanges(t *testing.T) {
oldNewCmdReporter := newCmdReporter
defer func() { newCmdReporter = oldNewCmdReporter }()
t.Run("public network selected", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
},
AddressRanges: nil, // should auto-detect
})
t.Run("discover public cidrs", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("public", "2000::1") +
"\n" + separator() + "\n" +
ipAddrOutput("public", "2000::1/112")
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.NoError(t, err)
assert.Equal(t, []string{"2000::/112"}, ranges)
})
t.Run("discover cluster cidrs", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
// no cmd reporter methods should be called b/c cluster net is not selected
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkCluster)
assert.NoError(t, err)
assert.Equal(t, []string{}, ranges)
})
t.Run("public ip mismatch", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("public", "192.168.1.1") +
"\n" + separator() + "\n" +
ipAddrOutput("public", "192.168.1.2/24") // IP is one more than above
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.Error(t, err)
assert.Equal(t, []string{}, ranges)
})
t.Run("public ip iface mismatch", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("public", "192.168.1.1") +
"\n" + separator() + "\n" +
ipAddrOutput("cluster", "192.168.1.1/24") // cluster != public above
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.Error(t, err)
assert.Equal(t, []string{}, ranges)
})
t.Run("iface wrong", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("cluster", "192.168.1.1") +
"\n" + separator() + "\n" +
ipAddrOutput("cluster", "192.168.1.1/24") // cluster != public above
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.Error(t, err)
assert.Equal(t, []string{}, ranges)
})
t.Run("no net status", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := "" + // no net status
"\n" + separator() + "\n" +
ipAddrOutput("public", "192.168.1.1/24")
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.Error(t, err)
assert.Equal(t, []string{}, ranges)
})
t.Run("no ip addr output", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("public", "2000::1") +
"\n" + separator() + "\n" +
"" // no ip addr output
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.Error(t, err)
assert.Equal(t, []string{}, ranges)
})
t.Run("cmdreporter run failure", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
cmdReporter.On("Run", mock.Anything, mock.Anything).Return("", "", 0, errors.New("fake err"))
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.Error(t, err)
assert.Equal(t, []string{}, ranges)
})
t.Run("cmdreporter nonzero exit", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
cmdReporter.On("Run", mock.Anything, mock.Anything).Return("some garbage", "", 1, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.Error(t, err)
assert.Equal(t, []string{}, ranges)
})
})
t.Run("cluster network selected", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: nil, // should auto-detect
})
t.Run("discover cluster cidrs", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("cluster", "10.144.1.5") +
"\n" + separator() + "\n" +
ipAddrOutput("cluster", "10.144.1.5/16")
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkCluster)
assert.NoError(t, err)
assert.Equal(t, []string{"10.144.0.0/16"}, ranges)
})
t.Run("discover public cidrs", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
// no cmd reporter methods should be called b/c public net is not selected
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.NoError(t, err)
assert.Equal(t, []string{}, ranges)
})
})
t.Run("public and cluster networks selected", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: nil, // should auto-detect
})
t.Run("discover public cidrs", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("public", "10.144.1.5") +
"\n" + separator() + "\n" +
ipAddrOutput("public", "10.144.1.5/16")
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.NoError(t, err)
assert.Equal(t, []string{"10.144.0.0/16"}, ranges)
})
t.Run("discover cluster cidrs", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("cluster", "1abc::1") +
"\n" + separator() + "\n" +
ipAddrOutput("cluster", "1abc::1/112")
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkCluster)
assert.NoError(t, err)
assert.Equal(t, []string{"1abc::/112"}, ranges)
})
t.Run("public multiple ips", func(t *testing.T) {
cmdReporter := new(mockCmdReporter)
cmdReporter.On("Job")
goodOutput := netStatus("public", "10.144.1.5", "fe05::6") +
"\n" + separator() + "\n" +
ipAddrOutput("public", "10.144.1.5/16", "fe05::6/96")
cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil)
newCmdReporter = mockNewCmdReporter(cmdReporter, nil)
ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic)
assert.NoError(t, err)
assert.Equal(t, []string{"10.144.0.0/16", "fe05::/96"}, ranges)
})
})
}
func mockDiscoverAddressRangesFunc(publicReturn []string, clusterReturn []string, panicIfDiscoverPublic, panicIfDiscoverCluster bool, shouldErr error) func(ctx context.Context, rookImage string, clusterdContext *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo, discoverPublic bool, discoverCluster bool) (publicRanges []cephv1.CIDR, clusterRanges []cephv1.CIDR, err error) {
return func(ctx context.Context, rookImage string, clusterdContext *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo, discoverPublic bool, discoverCluster bool) (publicRanges []cephv1.CIDR, clusterRanges []cephv1.CIDR, err error) {
// panic is fine for unit testing
if panicIfDiscoverPublic && discoverPublic {
panic("should not discover public net")
}
if panicIfDiscoverCluster && discoverCluster {
panic("should not discover cluster net")
}
return toCephCIDRs(publicReturn), toCephCIDRs(clusterReturn), shouldErr
}
}
type mockMonStore struct {
mock.Mock
}
func (m *mockMonStore) SetIfChanged(who string, option string, value string) (bool, error) {
args := m.Called(who, option, value)
return args.Bool(0), args.Error(1)
}
func mockGetMonStoreFunc(m *mockMonStore) func(context *clusterd.Context, clusterInfo *client.ClusterInfo) monStoreInterface {
return func(context *clusterd.Context, clusterInfo *client.ClusterInfo) monStoreInterface {
return m
}
}
func TestApplyCephNetworkSettings(t *testing.T) {
oldDiscoverCephAddressRangesFunc := discoverCephAddressRangesFunc
defer func() { discoverCephAddressRangesFunc = oldDiscoverCephAddressRangesFunc }()
oldGetMonStoreFunc := getMonStoreFunc
defer func() { getMonStoreFunc = oldGetMonStoreFunc }()
t.Run("no net defined", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(cephv1.NetworkSpec{})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, false, false, errors.New("should not be called"))
monStore := new(mockMonStore)
// implicit: mon store should not have SetIfChanged called on it
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("multus", func(t *testing.T) {
t.Run("public selected", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
},
AddressRanges: nil, // should auto-detect
})
t.Run("single public cidr", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/16"}, []string{}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/16").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("multiple public cidrs", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/16", "feab::/112"}, []string{}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/16, feab::/112").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
})
t.Run("cluster selected", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: nil, // should auto-detect
})
t.Run("single cluster cidr", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{"192.168.0.0/16"}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.0.0/16").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("multiple cluster cidrs", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{"192.168.0.0/16", "feab::/112"}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.0.0/16, feab::/112").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
})
t.Run("public and cluster selected", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: nil, // should auto-detect
})
t.Run("single cidr", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/16"}, []string{"192.168.10.0/16"}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/16").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.10.0/16").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("multiple cidrs", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/16", "feab::/112"}, []string{"192.168.10.0/16", "feab:10::/112"}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/16, feab::/112").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.10.0/16, feab:10::/112").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("single and multiple cidrs", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/16"}, []string{"192.168.10.0/16", "feab:10::/112"}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/16").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.10.0/16, feab:10::/112").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("multiple and single cidrs", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/16", "feab::/112"}, []string{"192.168.10.0/16"}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/16, feab::/112").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.10.0/16").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("discover err", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, false, false, errors.New("fake err"))
monStore := new(mockMonStore)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.ErrorContains(t, err, "fake err")
})
t.Run("public apply err", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/16"}, []string{"192.168.10.0/16"}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/16").Return(true, errors.New("fake err"))
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.10.0/16").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.ErrorContains(t, err, "fake err")
})
t.Run("cluster apply err", func(t *testing.T) {
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/16"}, []string{"192.168.10.0/16"}, false, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/16").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.10.0/16").Return(true, errors.New("fake err"))
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.ErrorContains(t, err, "fake err")
})
})
t.Run("public selected with address ranges", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
},
AddressRanges: &cephv1.AddressRangesSpec{
Public: []cephv1.CIDR{
"192.168.0.0/24",
},
},
})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, true, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/24").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("cluster selected with address ranges", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: &cephv1.AddressRangesSpec{
Cluster: []cephv1.CIDR{
"192.168.0.0/24",
},
},
})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, false, true, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.0.0/24").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("public and cluster selected with address ranges", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: &cephv1.AddressRangesSpec{
Public: []cephv1.CIDR{
"192.168.0.0/24",
},
Cluster: []cephv1.CIDR{
"192.168.50.0/24",
},
},
})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, true, true, errors.New("should not be called"))
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/24").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.50.0/24").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("public and cluster selected with multi-cidr address ranges", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: &cephv1.AddressRangesSpec{
Public: []cephv1.CIDR{
"192.168.0.0/24",
"fabc::/112",
},
Cluster: []cephv1.CIDR{
"192.168.50.0/24",
"fabc:50::/112",
},
},
})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, true, true, errors.New("should not be called"))
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/24, fabc::/112").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.50.0/24, fabc:50::/112").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("public and cluster selected, only cluster addr ranges specified", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: &cephv1.AddressRangesSpec{
Cluster: []cephv1.CIDR{
"192.168.50.0/24",
},
},
})
clusterAddrsShouldNotBeDiscovered := true
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{"192.168.0.0/24"}, []string{}, false, clusterAddrsShouldNotBeDiscovered, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/24").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.50.0/24").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("public and cluster selected, only cluster public ranges specified", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderMultus,
Selectors: map[cephv1.CephNetworkType]string{
cephv1.CephNetworkPublic: "macvlan-public",
cephv1.CephNetworkCluster: "macvlan-cluster",
},
AddressRanges: &cephv1.AddressRangesSpec{
Public: []cephv1.CIDR{
"192.168.0.0/24",
},
},
})
publicAddrsShouldNotBeDiscovered := true
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{"192.168.50.0/24"}, publicAddrsShouldNotBeDiscovered, false, nil)
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/24").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.50.0/24").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
})
t.Run("hostnet", func(t *testing.T) {
t.Run("no ranges specified", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderHost,
Selectors: map[cephv1.CephNetworkType]string{},
AddressRanges: nil,
})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, false, false, errors.New("should not be called"))
monStore := new(mockMonStore)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("public ranges specified", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderHost,
Selectors: map[cephv1.CephNetworkType]string{},
AddressRanges: &cephv1.AddressRangesSpec{
Public: []cephv1.CIDR{
"192.168.0.0/24",
"feab::/112",
},
},
})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, false, false, errors.New("should not be called"))
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/24, feab::/112").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("cluster ranges specified", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderHost,
Selectors: map[cephv1.CephNetworkType]string{},
AddressRanges: &cephv1.AddressRangesSpec{
Cluster: []cephv1.CIDR{
"192.168.0.0/24",
"feab::/112",
},
},
})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, false, false, errors.New("should not be called"))
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.0.0/24, feab::/112").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
t.Run("public and cluster ranges specified", func(t *testing.T) {
clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec(
cephv1.NetworkSpec{
Provider: cephv1.NetworkProviderHost,
Selectors: map[cephv1.CephNetworkType]string{},
AddressRanges: &cephv1.AddressRangesSpec{
Public: []cephv1.CIDR{
"192.168.0.0/24",
"feab::/112",
},
Cluster: []cephv1.CIDR{
"192.168.100.0/24",
"feab:100::/112",
},
},
})
discoverCephAddressRangesFunc = mockDiscoverAddressRangesFunc([]string{}, []string{}, false, false, errors.New("should not be called"))
monStore := new(mockMonStore)
monStore.On("SetIfChanged", "global", "public_network", "192.168.0.0/24, feab::/112").Return(true, nil)
monStore.On("SetIfChanged", "global", "cluster_network", "192.168.100.0/24, feab:100::/112").Return(true, nil)
getMonStoreFunc = mockGetMonStoreFunc(monStore)
err := ApplyCephNetworkSettings(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo)
assert.NoError(t, err)
})
})
}
// generate a net status for interface (expected to be "public" or "cluster") with some number of IPs
func netStatus(iface string, ips ...string) string {
ipsInJson := ""
for _, ip := range ips {
if ipsInJson != "" {
ipsInJson += ","
}
ipsInJson = ipsInJson + "\n" + ` "` + ip + `"`
}
out := `[{
"name": "bridge",
"interface": "eth0",
"ips": [
"10.244.1.152"
],
"mac": "1a:15:5d:61:18:50",
"default": true,
"dns": {},
"gateway": [
"10.244.0.1"
]
},{
"name": "dont/care",
"interface": "` + iface + `",
"ips": [` + ipsInJson + `
],
"mac": "8e:ec:4a:e3:1b:96",
"dns": {}
}]`
return out
}
// generate 'ip --json address show dev <iface>' output with some number of <ip>/<prefix>-es
// also contains extra IPv6 SLAAC addrs that are often present in containers
func ipAddrOutput(iface string, ipsWithPrefixLens ...string) string {
ipsInJson := ""
for _, ipWithPrefix := range ipsWithPrefixLens {
s := strings.Split(ipWithPrefix, "/")
ip := s[0]
prefix := s[1]
// specifically omit "family": "inet" from the output so it's certain that field isn't used
// when parsing IPv4 vs IPv6 addrs
ipsInJson = ipsInJson + `
{
"local": "` + ip + `",
"prefixlen": ` + prefix + `,
"scope": "global",
"valid_life_time": 4294967295,
"preferred_life_time": 4294967295
},`
}
out := `[
{
"ifindex": 3,
"link_index": 2,
"ifname": "` + iface + `",
"flags": [
"BROADCAST",
"MULTICAST",
"UP",
"LOWER_UP"
],
"mtu": 1500,
"qdisc": "noqueue",
"operstate": "UP",
"group": "default",
"link_type": "ether",
"address": "8e:ec:4a:e3:1b:96",
"broadcast": "ff:ff:ff:ff:ff:ff",
"link_netnsid": 0,
"addr_info": [
{
"family": "inet6",
"local": "fd4e:7658:764f:15c4:8cec:4aff:fee3:1b96",
"prefixlen": 64,
"scope": "global",
"dynamic": true,
"mngtmpaddr": true,
"valid_life_time": 2591998,
"preferred_life_time": 604798
},` + ipsInJson + `
{
"family": "inet6",
"local": "fe80::8cec:4aff:fee3:1b96",
"prefixlen": 64,
"scope": "link",
"valid_life_time": 4294967295,
"preferred_life_time": 4294967295
}
]
}
]`
return out
}
View Git Blame Copy Permalink