assignMons schedules each mon in its own goroutine and shares a failedMonSchedule bool to record whether any of them failed. The resultLock mutex already guards the shared c.mapping.Schedule map write, but the three failedMonSchedule = true assignments were done without holding the lock. When two or more mons fail to schedule at the same time (for example when waitForMonitorScheduling errors, the node choice is nil, or getNodeInfoFromNode fails) their goroutines write the flag concurrently, which is a write-write data race. Guard the failedMonSchedule writes with the existing resultLock, the same lock the goroutines already use for the map update. The post-Wait read is left as is since the WaitGroup orders it after every write. Add a regression test that fails scheduling for multiple mons at once and confirms assignMons returns an error; run it with -race to catch the race. Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com>
1649 lines
60 KiB
Go
1649 lines
60 KiB
Go
/*
|
|
Copyright 2016 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package mon
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
csiopv1 "github.com/ceph/ceph-csi-operator/api/v1"
|
|
"github.com/coreos/pkg/capnslog"
|
|
"github.com/pkg/errors"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
|
clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test"
|
|
"github.com/rook/rook/pkg/operator/ceph/config"
|
|
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
|
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
|
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
"github.com/rook/rook/pkg/operator/test"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
apps "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
discoveryv1 "k8s.io/api/discovery/v1"
|
|
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
)
|
|
|
|
// generate a standard mon config from a mon id w/ default port and IP 2.4.6.{1,2,3,...}
|
|
// support mon ID as new ["a", "b", etc.] form or as legacy ["mon0", "mon1", etc.] form
|
|
func testGenMonConfig(monID string) *monConfig {
|
|
var moniker string
|
|
var index int
|
|
var err error
|
|
if strings.HasPrefix(monID, "mon") { // is legacy mon name
|
|
moniker = monID // keep legacy "mon#" name
|
|
index, err = strconv.Atoi(strings.Replace(monID, "mon", "", 1)) // get # off end of mon#
|
|
} else {
|
|
moniker = "mon-" + monID
|
|
index, err = k8sutil.NameToIndex(monID)
|
|
}
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return &monConfig{
|
|
ResourceName: "rook-ceph-" + moniker, // rook-ceph-mon-A or rook-ceph-mon#
|
|
DaemonName: monID, // A or mon#
|
|
Port: DefaultMsgr1Port,
|
|
PublicIP: fmt.Sprintf("2.4.6.%d", index+1),
|
|
// dataDirHostPath assumed to be /var/lib/rook
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap(
|
|
"/var/lib/rook", dataDirRelativeHostPath(monID), config.MonType, monID, "rook-ceph"),
|
|
}
|
|
}
|
|
|
|
func newTestStartCluster(t *testing.T, namespace string) (*clusterd.Context, error) {
|
|
monResponse := func() (string, error) {
|
|
return clienttest.MonInQuorumResponseMany(3), nil
|
|
}
|
|
return newTestStartClusterWithQuorumResponse(t, namespace, monResponse)
|
|
}
|
|
|
|
func newTestStartClusterWithQuorumResponse(t *testing.T, namespace string, monResponse func() (string, error)) (*clusterd.Context, error) {
|
|
clientset := test.New(t, 3)
|
|
configDir := t.TempDir()
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if strings.Contains(command, "ceph-authtool") {
|
|
err := clienttest.CreateConfigDir(path.Join(configDir, namespace))
|
|
return "", errors.Wrap(err, "failed testing of start cluster without quorum response")
|
|
} else {
|
|
return monResponse()
|
|
}
|
|
},
|
|
}
|
|
s := scheme.Scheme
|
|
assert.NoError(t, csiopv1.AddToScheme(s))
|
|
cl := fake.NewClientBuilder().WithScheme(s).Build()
|
|
return &clusterd.Context{
|
|
Clientset: clientset,
|
|
Executor: executor,
|
|
ConfigDir: configDir,
|
|
Client: cl,
|
|
}, nil
|
|
}
|
|
|
|
func newCluster(context *clusterd.Context, namespace string, allowMultiplePerNode bool, resources v1.ResourceRequirements) *Cluster {
|
|
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
|
|
return &Cluster{
|
|
ClusterInfo: nil,
|
|
context: context,
|
|
Namespace: namespace,
|
|
rookImage: "myversion",
|
|
spec: cephv1.ClusterSpec{
|
|
Mon: cephv1.MonSpec{
|
|
Count: 3,
|
|
AllowMultiplePerNode: allowMultiplePerNode,
|
|
},
|
|
Resources: map[string]v1.ResourceRequirements{"mon": resources},
|
|
},
|
|
maxMonID: -1,
|
|
waitForStart: false,
|
|
monTimeoutList: map[string]time.Time{},
|
|
mapping: &opcontroller.Mapping{
|
|
Schedule: map[string]*opcontroller.MonScheduleInfo{},
|
|
},
|
|
ownerInfo: ownerInfo,
|
|
monsToFailover: map[string]*monConfig{},
|
|
}
|
|
}
|
|
|
|
// setCommonMonProperties is a convenience helper for setting common test properties
|
|
func setCommonMonProperties(c *Cluster, currentMons int, mon cephv1.MonSpec, rookImage string) {
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(currentMons)
|
|
c.spec.Mon.Count = mon.Count
|
|
c.spec.Mon.AllowMultiplePerNode = mon.AllowMultiplePerNode
|
|
c.rookImage = rookImage
|
|
}
|
|
|
|
func TestResourceName(t *testing.T) {
|
|
assert.Equal(t, "rook-ceph-mon-a", resourceName("rook-ceph-mon-a"))
|
|
assert.Equal(t, "rook-ceph-mon123", resourceName("rook-ceph-mon123"))
|
|
assert.Equal(t, "rook-ceph-mon-b", resourceName("b"))
|
|
}
|
|
|
|
func TestStartMonDeployment(t *testing.T) {
|
|
namespace := "ns"
|
|
context, err := newTestStartCluster(t, namespace)
|
|
assert.NoError(t, err)
|
|
c := newCluster(context, namespace, true, v1.ResourceRequirements{})
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
|
|
cm := &v1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: EndpointConfigMapName},
|
|
Data: map[string]string{"maxMonId": "1"},
|
|
}
|
|
_, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(c.ClusterInfo.Context, cm, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
// Start mon a on a specific node since there is no volumeClaimTemplate
|
|
m := &monConfig{ResourceName: "rook-ceph-mon-a", DaemonName: "a", Port: 3300, PublicIP: "1.2.3.4", DataPathMap: &config.DataPathMap{}}
|
|
schedule := &opcontroller.MonScheduleInfo{Hostname: "host-a", Zone: "zonea"}
|
|
err = c.startMon(m, schedule)
|
|
assert.NoError(t, err)
|
|
deployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, schedule.Hostname, deployment.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"])
|
|
|
|
// Start mon b on any node in a zone since there is a volumeClaimTemplate
|
|
m = &monConfig{ResourceName: "rook-ceph-mon-b", DaemonName: "b", Port: 3300, PublicIP: "1.2.3.5", DataPathMap: &config.DataPathMap{}}
|
|
schedule = &opcontroller.MonScheduleInfo{Hostname: "host-b", Zone: "zoneb"}
|
|
c.spec.Mon.VolumeClaimTemplate = &cephv1.VolumeClaimTemplate{}
|
|
err = c.startMon(m, schedule)
|
|
assert.NoError(t, err)
|
|
deployment, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
// no node selector when there is a volumeClaimTemplate and the mon is assigned to a zone
|
|
assert.Equal(t, 0, len(deployment.Spec.Template.Spec.NodeSelector))
|
|
}
|
|
|
|
func TestStartMonPods(t *testing.T) {
|
|
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
|
|
os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
|
|
namespace := "ns"
|
|
context, err := newTestStartCluster(t, namespace)
|
|
assert.NoError(t, err)
|
|
c := newCluster(context, namespace, true, v1.ResourceRequirements{})
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
c.spec.Annotations = cephv1.AnnotationsSpec{
|
|
cephv1.KeyClusterMetadata: cephv1.Annotations{
|
|
"key": "value",
|
|
},
|
|
}
|
|
|
|
// start a basic cluster
|
|
_, err = c.Start(c.ClusterInfo, c.rookImage, cephver.Squid, c.spec)
|
|
assert.NoError(t, err)
|
|
|
|
// test annotations
|
|
secret, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, map[string]string{"key": "value"}, secret.Annotations)
|
|
|
|
validateStart(t, c)
|
|
|
|
// starting again should be a no-op
|
|
_, err = c.Start(c.ClusterInfo, c.rookImage, cephver.Squid, c.spec)
|
|
assert.NoError(t, err)
|
|
|
|
validateStart(t, c)
|
|
}
|
|
|
|
func TestOperatorRestart(t *testing.T) {
|
|
namespace := "ns"
|
|
context, err := newTestStartCluster(t, namespace)
|
|
assert.NoError(t, err)
|
|
c := newCluster(context, namespace, true, v1.ResourceRequirements{})
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
|
|
// start a basic cluster
|
|
info, err := c.Start(c.ClusterInfo, c.rookImage, cephver.Squid, c.spec)
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, info.IsInitialized())
|
|
|
|
validateStart(t, c)
|
|
|
|
c = newCluster(context, namespace, true, v1.ResourceRequirements{})
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
|
|
// starting again should be a no-op, but will not result in an error
|
|
info, err = c.Start(c.ClusterInfo, c.rookImage, cephver.Squid, c.spec)
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, info.IsInitialized())
|
|
|
|
validateStart(t, c)
|
|
}
|
|
|
|
// safety check that if hostNetwork is used no changes occur on an operator restart
|
|
func TestOperatorRestartHostNetwork(t *testing.T) {
|
|
namespace := "ns"
|
|
context, err := newTestStartCluster(t, namespace)
|
|
assert.NoError(t, err)
|
|
|
|
// cluster without host networking
|
|
c := newCluster(context, namespace, false, v1.ResourceRequirements{})
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
|
|
// start a basic cluster
|
|
info, err := c.Start(c.ClusterInfo, c.rookImage, cephver.Squid, c.spec)
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, info.IsInitialized())
|
|
|
|
validateStart(t, c)
|
|
|
|
// cluster with host networking
|
|
c = newCluster(context, namespace, false, v1.ResourceRequirements{})
|
|
c.spec.Network.HostNetwork = true
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
|
|
// starting again should be a no-op, but still results in an error
|
|
info, err = c.Start(c.ClusterInfo, c.rookImage, cephver.Squid, c.spec)
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, info.IsInitialized(), info)
|
|
|
|
validateStart(t, c)
|
|
}
|
|
|
|
func validateStart(t *testing.T, c *Cluster) {
|
|
s, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{})
|
|
assert.NoError(t, err) // there shouldn't be an error due the secret existing
|
|
assert.Equal(t, 4, len(s.Data))
|
|
|
|
// there is only one pod created. the other two won't be created since the first one doesn't start
|
|
_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, "rook-ceph-mon-a", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
func TestPersistMons(t *testing.T) {
|
|
clientset := test.New(t, 1)
|
|
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
|
|
c := New(context.TODO(), &clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{Annotations: cephv1.AnnotationsSpec{cephv1.KeyClusterMetadata: cephv1.Annotations{"key": "value"}}}, ownerInfo)
|
|
setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
|
|
|
expectedPorts := []discoveryv1.EndpointPort{
|
|
{Name: ptr.To(DefaultMsgr2PortName), Protocol: ptr.To(v1.ProtocolTCP), Port: ptr.To(DefaultMsgr2Port)},
|
|
{Name: ptr.To(DefaultMsgr1PortName), Protocol: ptr.To(v1.ProtocolTCP), Port: ptr.To(DefaultMsgr1Port)},
|
|
}
|
|
|
|
// Persist mon a
|
|
err := c.persistExpectedMonDaemonsInConfigMap()
|
|
assert.NoError(t, err)
|
|
err = c.persistExpectedMonDaemonsAsEndpointSlice()
|
|
assert.NoError(t, err)
|
|
|
|
cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "a=1.2.3.1:3300", cm.Data[EndpointDataKey])
|
|
assert.Equal(t, map[string]string{"key": "value"}, cm.Annotations)
|
|
|
|
ep, err := c.context.Clientset.DiscoveryV1().EndpointSlices(c.Namespace).Get(context.TODO(), endpointSliceNameIPv4, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "1.2.3.1", ep.Endpoints[0].Addresses[0])
|
|
assert.ElementsMatch(t, expectedPorts, ep.Ports)
|
|
assert.Equal(t, map[string]string{"key": "value"}, cm.Annotations)
|
|
|
|
// Persist mon b, and remove mon a for simply testing the configmap is updated
|
|
c.ClusterInfo.InternalMonitors["b"] = &cephclient.MonInfo{Name: "b", Endpoint: "4.5.6.7:3300"}
|
|
delete(c.ClusterInfo.InternalMonitors, "a")
|
|
err = c.persistExpectedMonDaemonsInConfigMap()
|
|
assert.NoError(t, err)
|
|
err = c.persistExpectedMonDaemonsAsEndpointSlice()
|
|
assert.NoError(t, err)
|
|
|
|
cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "b=4.5.6.7:3300", cm.Data[EndpointDataKey])
|
|
|
|
ep, err = c.context.Clientset.DiscoveryV1().EndpointSlices(c.Namespace).Get(context.TODO(), endpointSliceNameIPv4, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "4.5.6.7", ep.Endpoints[0].Addresses[0])
|
|
assert.ElementsMatch(t, expectedPorts, ep.Ports)
|
|
assert.Equal(t, map[string]string{"key": "value"}, cm.Annotations)
|
|
}
|
|
|
|
func TestCreateEndpointSlices(t *testing.T) {
|
|
clientset := test.New(t, 1)
|
|
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
|
|
|
|
// RequireMsgr2=false
|
|
c := New(context.TODO(), &clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, ownerInfo)
|
|
expectedPorts := []discoveryv1.EndpointPort{
|
|
{Name: ptr.To(DefaultMsgr2PortName), Protocol: ptr.To(v1.ProtocolTCP), Port: ptr.To(DefaultMsgr2Port)},
|
|
{Name: ptr.To(DefaultMsgr1PortName), Protocol: ptr.To(v1.ProtocolTCP), Port: ptr.To(DefaultMsgr1Port)},
|
|
}
|
|
testCreateEndpointSlicesForCluster(t, c, expectedPorts)
|
|
|
|
// RequireMsgr2=true
|
|
c = New(
|
|
context.TODO(),
|
|
&clusterd.Context{Clientset: clientset},
|
|
"ns",
|
|
cephv1.ClusterSpec{
|
|
Network: cephv1.NetworkSpec{
|
|
Connections: &cephv1.ConnectionsSpec{
|
|
RequireMsgr2: true,
|
|
},
|
|
},
|
|
}, ownerInfo)
|
|
expectedPorts = []discoveryv1.EndpointPort{
|
|
{Name: ptr.To(DefaultMsgr2PortName), Protocol: ptr.To(v1.ProtocolTCP), Port: ptr.To(DefaultMsgr2Port)},
|
|
}
|
|
testCreateEndpointSlicesForCluster(t, c, expectedPorts)
|
|
}
|
|
|
|
func testCreateEndpointSlicesForCluster(t *testing.T, c *Cluster, expectedPorts []discoveryv1.EndpointPort) {
|
|
ipv4Addresses := []string{"10.0.0.1", "10.0.0.2", "10.0.0.3"}
|
|
ipv4Mons := []cephclient.MonInfo{
|
|
{Name: "a", Endpoint: fmt.Sprintf("%s:6789", ipv4Addresses[0])},
|
|
{Name: "b", Endpoint: fmt.Sprintf("%s:6789", ipv4Addresses[1])},
|
|
{Name: "c", Endpoint: fmt.Sprintf("%s:6789", ipv4Addresses[2])},
|
|
}
|
|
ipv6Addresses := []string{"2001:db8::1", "2001:db8::2", "2001:db8::3"}
|
|
ipv6Mons := []cephclient.MonInfo{
|
|
{Name: "d", Endpoint: fmt.Sprintf("[%s]:6789", ipv6Addresses[0])},
|
|
{Name: "e", Endpoint: fmt.Sprintf("[%s]:6789", ipv6Addresses[1])},
|
|
{Name: "f", Endpoint: fmt.Sprintf("[%s]:6789", ipv6Addresses[2])},
|
|
}
|
|
|
|
// IPv4 test
|
|
c.ClusterInfo.InternalMonitors = map[string]*cephclient.MonInfo{}
|
|
for _, mon := range ipv4Mons {
|
|
c.ClusterInfo.InternalMonitors[mon.Name] = &mon
|
|
}
|
|
|
|
err := c.persistExpectedMonDaemonsAsEndpointSlice()
|
|
assert.NoError(t, err)
|
|
|
|
epSliceIPv4, err := c.context.Clientset.DiscoveryV1().EndpointSlices(c.Namespace).Get(context.TODO(), endpointSliceNameIPv4, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, discoveryv1.AddressTypeIPv4, epSliceIPv4.AddressType)
|
|
assert.Len(t, epSliceIPv4.Endpoints, 1)
|
|
assert.ElementsMatch(t, ipv4Addresses, epSliceIPv4.Endpoints[0].Addresses)
|
|
assert.ElementsMatch(t, expectedPorts, epSliceIPv4.Ports)
|
|
|
|
_, err = c.context.Clientset.DiscoveryV1().EndpointSlices(c.Namespace).Get(context.TODO(), endpointSliceNameIPv6, metav1.GetOptions{})
|
|
assert.True(t, kerrors.IsNotFound(err))
|
|
|
|
// IPv6 test
|
|
c.ClusterInfo.InternalMonitors = map[string]*cephclient.MonInfo{}
|
|
for _, mon := range ipv6Mons {
|
|
c.ClusterInfo.InternalMonitors[mon.Name] = &mon
|
|
}
|
|
|
|
err = c.persistExpectedMonDaemonsAsEndpointSlice()
|
|
assert.NoError(t, err)
|
|
|
|
_, err = c.context.Clientset.DiscoveryV1().EndpointSlices(c.Namespace).Get(context.TODO(), endpointSliceNameIPv4, metav1.GetOptions{})
|
|
assert.True(t, kerrors.IsNotFound(err))
|
|
|
|
epSliceIPv6, err := c.context.Clientset.DiscoveryV1().EndpointSlices(c.Namespace).Get(context.TODO(), endpointSliceNameIPv6, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, discoveryv1.AddressTypeIPv6, epSliceIPv6.AddressType)
|
|
assert.Len(t, epSliceIPv6.Endpoints, 1)
|
|
assert.ElementsMatch(t, ipv6Addresses, epSliceIPv6.Endpoints[0].Addresses)
|
|
assert.ElementsMatch(t, expectedPorts, epSliceIPv6.Ports)
|
|
|
|
// Mixed IPv4 and IPv6 test.
|
|
// Note that this normally doesn't happen, because rook uses only IPv4 or
|
|
// only IPv6 mons. But in the future, migration between IPv4 and IPv6 might
|
|
// be supported, and then, during migration, the old mons would use the
|
|
// old IP family and the new mons would use the new IP family.
|
|
c.ClusterInfo.InternalMonitors = map[string]*cephclient.MonInfo{}
|
|
for _, mon := range ipv4Mons {
|
|
c.ClusterInfo.InternalMonitors[mon.Name] = &mon
|
|
}
|
|
for _, mon := range ipv6Mons {
|
|
c.ClusterInfo.InternalMonitors[mon.Name] = &mon
|
|
}
|
|
|
|
err = c.persistExpectedMonDaemonsAsEndpointSlice()
|
|
assert.NoError(t, err)
|
|
|
|
epSliceIPv4, err = c.context.Clientset.DiscoveryV1().EndpointSlices(c.Namespace).Get(context.TODO(), endpointSliceNameIPv4, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, discoveryv1.AddressTypeIPv4, epSliceIPv4.AddressType)
|
|
assert.Len(t, epSliceIPv4.Endpoints, 1)
|
|
assert.ElementsMatch(t, ipv4Addresses, epSliceIPv4.Endpoints[0].Addresses)
|
|
assert.ElementsMatch(t, expectedPorts, epSliceIPv4.Ports)
|
|
|
|
epSliceIPv6, err = c.context.Clientset.DiscoveryV1().EndpointSlices(c.Namespace).Get(context.TODO(), endpointSliceNameIPv6, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, discoveryv1.AddressTypeIPv6, epSliceIPv6.AddressType)
|
|
assert.Len(t, epSliceIPv6.Endpoints, 1)
|
|
assert.ElementsMatch(t, ipv6Addresses, epSliceIPv6.Endpoints[0].Addresses)
|
|
assert.ElementsMatch(t, expectedPorts, epSliceIPv6.Ports)
|
|
}
|
|
|
|
func TestSaveMonEndpoints(t *testing.T) {
|
|
ctx := context.TODO()
|
|
clientset := test.New(t, 1)
|
|
configDir := t.TempDir()
|
|
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
|
|
s := scheme.Scheme
|
|
assert.NoError(t, csiopv1.AddToScheme(s))
|
|
cl := fake.NewClientBuilder().WithScheme(s).Build()
|
|
c := New(ctx, &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Client: cl}, "ns", cephv1.ClusterSpec{}, ownerInfo)
|
|
setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
|
|
|
// create the initial config map
|
|
err := c.saveMonConfig()
|
|
assert.NoError(t, err)
|
|
|
|
cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "a=1.2.3.1:3300", cm.Data[EndpointDataKey])
|
|
assert.Equal(t, `{"node":{}}`, cm.Data[opcontroller.MappingKey])
|
|
assert.Equal(t, "-1", cm.Data[opcontroller.MaxMonIDKey])
|
|
|
|
// update the config map
|
|
c.ClusterInfo.InternalMonitors["a"].Endpoint = "2.3.4.5:6789"
|
|
c.maxMonID = 2
|
|
c.mapping.Schedule["a"] = &opcontroller.MonScheduleInfo{
|
|
Name: "node0",
|
|
Address: "1.1.1.1",
|
|
Hostname: "myhost",
|
|
}
|
|
err = c.saveMonConfig()
|
|
assert.NoError(t, err)
|
|
|
|
cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "a=2.3.4.5:6789", cm.Data[EndpointDataKey])
|
|
assert.Equal(t, `{"node":{"a":{"Name":"node0","Hostname":"myhost","Address":"1.1.1.1"}}}`, cm.Data[opcontroller.MappingKey])
|
|
assert.Equal(t, "-1", cm.Data[opcontroller.MaxMonIDKey])
|
|
|
|
// Update the maxMonID to some random value
|
|
cm.Data[opcontroller.MaxMonIDKey] = "23"
|
|
_, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(ctx, cm, metav1.UpdateOptions{})
|
|
assert.NoError(t, err)
|
|
// Confirm the maxMonId will be persisted and not updated to anything else.
|
|
// The value is only expected to be set directly to the configmap when a mon deployment is started.
|
|
err = c.saveMonConfig()
|
|
assert.NoError(t, err)
|
|
cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "23", cm.Data[opcontroller.MaxMonIDKey])
|
|
}
|
|
|
|
func TestMaxMonID(t *testing.T) {
|
|
clientset := test.New(t, 1)
|
|
configDir := t.TempDir()
|
|
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
|
|
s := scheme.Scheme
|
|
assert.NoError(t, csiopv1.AddToScheme(s))
|
|
cl := fake.NewClientBuilder().WithScheme(s).Build()
|
|
c := New(context.TODO(), &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Client: cl}, "ns", cephv1.ClusterSpec{}, ownerInfo)
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
|
|
// when the configmap is not found, the maxMonID is -1
|
|
maxMonID, err := c.getStoredMaxMonID()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "-1", maxMonID)
|
|
|
|
// initialize the configmap
|
|
setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
|
err = c.saveMonConfig()
|
|
assert.NoError(t, err)
|
|
|
|
// invalid mon names won't update the maxMonID
|
|
err = c.commitMaxMonID("bad-id")
|
|
assert.Error(t, err)
|
|
|
|
// starting a mon deployment will set the maxMonID
|
|
err = c.commitMaxMonID("a")
|
|
assert.NoError(t, err)
|
|
maxMonID, err = c.getStoredMaxMonID()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "0", maxMonID)
|
|
|
|
// set to a higher id
|
|
err = c.commitMaxMonID("d")
|
|
assert.NoError(t, err)
|
|
maxMonID, err = c.getStoredMaxMonID()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "3", maxMonID)
|
|
|
|
// setting to an id lower than the max will not update it
|
|
err = c.commitMaxMonID("c")
|
|
assert.NoError(t, err)
|
|
maxMonID, err = c.getStoredMaxMonID()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "3", maxMonID)
|
|
}
|
|
|
|
func TestMonInQuorum(t *testing.T) {
|
|
entry := cephclient.MonMapEntry{Name: "foo", Rank: 23}
|
|
quorum := []int{}
|
|
// Nothing in quorum
|
|
assert.False(t, monInQuorum(entry, quorum))
|
|
|
|
// One or more members in quorum
|
|
quorum = []int{23}
|
|
assert.True(t, monInQuorum(entry, quorum))
|
|
quorum = []int{5, 6, 7, 23, 8}
|
|
assert.True(t, monInQuorum(entry, quorum))
|
|
|
|
// Not in quorum
|
|
entry.Rank = 1
|
|
assert.False(t, monInQuorum(entry, quorum))
|
|
}
|
|
|
|
func TestNameToIndex(t *testing.T) {
|
|
// invalid
|
|
id, err := fullNameToIndex("rook-ceph-monitor0")
|
|
assert.NotNil(t, err)
|
|
assert.Equal(t, -1, id)
|
|
id, err = fullNameToIndex("rook-ceph-mon123")
|
|
assert.NotNil(t, err)
|
|
assert.Equal(t, -1, id)
|
|
|
|
// valid
|
|
id, err = fullNameToIndex("b")
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 1, id)
|
|
id, err = fullNameToIndex("m")
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 12, id)
|
|
id, err = fullNameToIndex("rook-ceph-mon-a")
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 0, id)
|
|
}
|
|
|
|
func TestWaitForQuorum(t *testing.T) {
|
|
namespace := "ns"
|
|
quorumChecks := 0
|
|
quorumResponse := func() (string, error) {
|
|
mons := map[string]*cephclient.MonInfo{
|
|
"a": {},
|
|
}
|
|
quorumChecks++
|
|
if quorumChecks == 1 {
|
|
// return an error the first time while we're waiting for the mon to join quorum
|
|
return "", errors.New("test error")
|
|
}
|
|
// a successful response indicates that we have quorum, even if we didn't check which specific mons were in quorum
|
|
return clienttest.MonInQuorumResponseFromMons(mons), nil
|
|
}
|
|
context, err := newTestStartClusterWithQuorumResponse(t, namespace, quorumResponse)
|
|
assert.NoError(t, err)
|
|
requireAllInQuorum := false
|
|
expectedMons := []string{"a"}
|
|
c := New(t.Context(), &clusterd.Context{Clientset: test.New(t, 1), ConfigDir: t.TempDir()}, "ns", cephv1.ClusterSpec{}, cephclient.NewMinimumOwnerInfoWithOwnerRef())
|
|
clusterInfo := cephclient.AdminTestClusterInfo("mycluster")
|
|
err = c.waitForQuorumWithMons(context, clusterInfo, expectedMons, 0, requireAllInQuorum)
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
func TestMonFoundInQuorum(t *testing.T) {
|
|
response := cephclient.MonStatusResponse{}
|
|
|
|
// "a" is in quorum
|
|
response.Quorum = []int{0}
|
|
response.MonMap.Mons = []cephclient.MonMapEntry{
|
|
{Name: "a", Rank: 0},
|
|
{Name: "b", Rank: 1},
|
|
{Name: "c", Rank: 2},
|
|
}
|
|
assert.True(t, monFoundInQuorum("a", response))
|
|
assert.False(t, monFoundInQuorum("b", response))
|
|
assert.False(t, monFoundInQuorum("c", response))
|
|
|
|
// b and c also in quorum, but not d
|
|
response.Quorum = []int{0, 1, 2}
|
|
assert.True(t, monFoundInQuorum("a", response))
|
|
assert.True(t, monFoundInQuorum("b", response))
|
|
assert.True(t, monFoundInQuorum("c", response))
|
|
assert.False(t, monFoundInQuorum("d", response))
|
|
}
|
|
|
|
func TestConfigureArbiter(t *testing.T) {
|
|
c := &Cluster{spec: cephv1.ClusterSpec{
|
|
Mon: cephv1.MonSpec{
|
|
StretchCluster: &cephv1.StretchClusterSpec{
|
|
Zones: []cephv1.MonZoneSpec{
|
|
{Name: "a", Arbiter: true},
|
|
{Name: "b"},
|
|
{Name: "c"},
|
|
},
|
|
},
|
|
},
|
|
}}
|
|
c.arbiterMon = "arb"
|
|
currentArbiter := c.arbiterMon
|
|
setNewTiebreaker := false
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
logger.Infof("%s %v", command, args)
|
|
if args[0] == "mon" {
|
|
if args[1] == "dump" {
|
|
return fmt.Sprintf(`{"tiebreaker_mon": "%s", "stretch_mode": true}`, currentArbiter), nil
|
|
}
|
|
if args[1] == "set_new_tiebreaker" {
|
|
assert.Equal(t, c.arbiterMon, args[2])
|
|
setNewTiebreaker = true
|
|
return "", nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("unrecognized output file command: %s %v", command, args)
|
|
},
|
|
}
|
|
c.context = &clusterd.Context{Clientset: test.New(t, 5), Executor: executor}
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(5)
|
|
|
|
t.Run("stretch mode already configured - new", func(t *testing.T) {
|
|
c.arbiterMon = currentArbiter
|
|
err := c.ConfigureArbiter()
|
|
assert.NoError(t, err)
|
|
assert.False(t, setNewTiebreaker)
|
|
})
|
|
t.Run("tiebreaker changed", func(t *testing.T) {
|
|
c.arbiterMon = "changed"
|
|
err := c.ConfigureArbiter()
|
|
assert.NoError(t, err)
|
|
assert.True(t, setNewTiebreaker)
|
|
})
|
|
}
|
|
|
|
func TestFindAvailableZoneMon(t *testing.T) {
|
|
c := &Cluster{spec: cephv1.ClusterSpec{
|
|
Mon: cephv1.MonSpec{
|
|
Zones: []cephv1.MonZoneSpec{
|
|
{Name: "a"},
|
|
{Name: "b"},
|
|
{Name: "c"},
|
|
},
|
|
},
|
|
}}
|
|
|
|
// No mons are assigned to a zone yet
|
|
existingMons := []*monConfig{}
|
|
availableZone, err := c.findAvailableZone(existingMons)
|
|
assert.NoError(t, err)
|
|
assert.NotEqual(t, "", availableZone)
|
|
|
|
// With 3 mons, we have one available zone
|
|
existingMons = []*monConfig{
|
|
{ResourceName: "x", Zone: "a"},
|
|
{ResourceName: "y", Zone: "b"},
|
|
}
|
|
c.spec.Mon.Count = 3
|
|
availableZone, err = c.findAvailableZone(existingMons)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "c", availableZone)
|
|
|
|
// With 3 mons and no available zones
|
|
existingMons = []*monConfig{
|
|
{ResourceName: "x", Zone: "a"},
|
|
{ResourceName: "y", Zone: "b"},
|
|
{ResourceName: "z", Zone: "c"},
|
|
}
|
|
c.spec.Mon.Count = 3
|
|
availableZone, err = c.findAvailableZone(existingMons)
|
|
assert.Error(t, err)
|
|
assert.Equal(t, "", availableZone)
|
|
}
|
|
|
|
func TestFindAvailableZoneForStretchedMon(t *testing.T) {
|
|
c := &Cluster{spec: cephv1.ClusterSpec{
|
|
Mon: cephv1.MonSpec{
|
|
StretchCluster: &cephv1.StretchClusterSpec{
|
|
Zones: []cephv1.MonZoneSpec{
|
|
{Name: "a", Arbiter: true},
|
|
{Name: "b"},
|
|
{Name: "c"},
|
|
},
|
|
},
|
|
},
|
|
}}
|
|
|
|
// No mons are assigned to a zone yet
|
|
existingMons := []*monConfig{}
|
|
availableZone, err := c.findAvailableZone(existingMons)
|
|
assert.NoError(t, err)
|
|
assert.NotEqual(t, "", availableZone)
|
|
|
|
// With 3 mons, we have one available zone
|
|
existingMons = []*monConfig{
|
|
{ResourceName: "x", Zone: "a"},
|
|
{ResourceName: "y", Zone: "b"},
|
|
}
|
|
c.spec.Mon.Count = 3
|
|
availableZone, err = c.findAvailableZone(existingMons)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "c", availableZone)
|
|
|
|
// With 3 mons and no available zones
|
|
existingMons = []*monConfig{
|
|
{ResourceName: "x", Zone: "a"},
|
|
{ResourceName: "y", Zone: "b"},
|
|
{ResourceName: "z", Zone: "c"},
|
|
}
|
|
c.spec.Mon.Count = 3
|
|
availableZone, err = c.findAvailableZone(existingMons)
|
|
assert.Error(t, err)
|
|
assert.Equal(t, "", availableZone)
|
|
|
|
// With 5 mons and no available zones
|
|
existingMons = []*monConfig{
|
|
{ResourceName: "w", Zone: "a"},
|
|
{ResourceName: "x", Zone: "b"},
|
|
{ResourceName: "y", Zone: "b"},
|
|
{ResourceName: "z", Zone: "c"},
|
|
{ResourceName: "q", Zone: "c"},
|
|
}
|
|
c.spec.Mon.Count = 5
|
|
availableZone, err = c.findAvailableZone(existingMons)
|
|
assert.Error(t, err)
|
|
assert.Equal(t, "", availableZone)
|
|
|
|
// With 5 mons and one available zone
|
|
existingMons = []*monConfig{
|
|
{ResourceName: "w", Zone: "a"},
|
|
{ResourceName: "x", Zone: "b"},
|
|
{ResourceName: "y", Zone: "b"},
|
|
{ResourceName: "z", Zone: "c"},
|
|
}
|
|
availableZone, err = c.findAvailableZone(existingMons)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "c", availableZone)
|
|
|
|
// With 5 mons and arbiter zone is available zone
|
|
existingMons = []*monConfig{
|
|
{ResourceName: "w", Zone: "b"},
|
|
{ResourceName: "x", Zone: "b"},
|
|
{ResourceName: "y", Zone: "c"},
|
|
{ResourceName: "z", Zone: "c"},
|
|
}
|
|
availableZone, err = c.findAvailableZone(existingMons)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "a", availableZone)
|
|
}
|
|
|
|
func TestMonVolumeClaimTemplate(t *testing.T) {
|
|
generalSC := "generalSC"
|
|
zoneSC := "zoneSC"
|
|
defaultTemplate := &cephv1.VolumeClaimTemplate{Spec: v1.PersistentVolumeClaimSpec{StorageClassName: &generalSC}}
|
|
zoneTemplate := &cephv1.VolumeClaimTemplate{Spec: v1.PersistentVolumeClaimSpec{StorageClassName: &zoneSC}}
|
|
type fields struct {
|
|
spec cephv1.ClusterSpec
|
|
}
|
|
type args struct {
|
|
mon *monConfig
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
args args
|
|
want *v1.PersistentVolumeClaim
|
|
}{
|
|
{"no template", fields{cephv1.ClusterSpec{}}, args{&monConfig{Zone: "z1"}}, nil},
|
|
{"default template", fields{cephv1.ClusterSpec{Mon: cephv1.MonSpec{VolumeClaimTemplate: defaultTemplate}}}, args{&monConfig{Zone: "z1"}}, defaultTemplate.ToPVC()},
|
|
{
|
|
"default template with 3 zones",
|
|
fields{cephv1.ClusterSpec{Mon: cephv1.MonSpec{
|
|
VolumeClaimTemplate: defaultTemplate,
|
|
Zones: []cephv1.MonZoneSpec{{Name: "z1"}, {Name: "z2"}, {Name: "z3"}},
|
|
}}},
|
|
args{&monConfig{Zone: "z1"}},
|
|
defaultTemplate.ToPVC(),
|
|
},
|
|
{
|
|
"overridden template",
|
|
fields{cephv1.ClusterSpec{Mon: cephv1.MonSpec{
|
|
VolumeClaimTemplate: defaultTemplate,
|
|
Zones: []cephv1.MonZoneSpec{{Name: "z1", VolumeClaimTemplate: zoneTemplate}, {Name: "z2"}, {Name: "z3"}},
|
|
}}},
|
|
args{&monConfig{Zone: "z1"}},
|
|
zoneTemplate.ToPVC(),
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
c := &Cluster{
|
|
spec: tt.fields.spec,
|
|
}
|
|
if got := c.monVolumeClaimTemplate(tt.args.mon); !reflect.DeepEqual(got, tt.want) {
|
|
t.Errorf("Cluster.monVolumeClaimTemplate() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRemoveExtraMonDeployments(t *testing.T) {
|
|
namespace := "ns"
|
|
context, err := newTestStartCluster(t, namespace)
|
|
assert.NoError(t, err)
|
|
c := newCluster(context, namespace, true, v1.ResourceRequirements{})
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
|
|
// Nothing to remove when the mons match the deployment
|
|
mons := []*monConfig{
|
|
{ResourceName: "rook-ceph-mon-a", DaemonName: "a"},
|
|
}
|
|
deployments := []apps.Deployment{
|
|
{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon-a",
|
|
Labels: map[string]string{"ceph_daemon_id": "a"},
|
|
},
|
|
},
|
|
}
|
|
c.spec.Mon.Count = 1
|
|
removed := c.checkForExtraMonResources(mons, deployments)
|
|
assert.Equal(t, "", removed)
|
|
|
|
// Remove an extra mon deployment
|
|
deployments = append(deployments, apps.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon-b",
|
|
Labels: map[string]string{"ceph_daemon_id": "b"},
|
|
},
|
|
})
|
|
removed = c.checkForExtraMonResources(mons, deployments)
|
|
assert.Equal(t, "b", removed)
|
|
|
|
// Nothing to remove when there are not enough deployments for the expected mons
|
|
mons = []*monConfig{
|
|
{ResourceName: "rook-ceph-mon-a", DaemonName: "a"},
|
|
{ResourceName: "rook-ceph-mon-b", DaemonName: "b"},
|
|
{ResourceName: "rook-ceph-mon-c", DaemonName: "c"},
|
|
}
|
|
c.spec.Mon.Count = 3
|
|
removed = c.checkForExtraMonResources(mons, deployments)
|
|
assert.Equal(t, "", removed)
|
|
|
|
// Do not remove a mon when it was during failover and only a single mon is in the list, even if extra deployments exist
|
|
mons = []*monConfig{
|
|
{ResourceName: "rook-ceph-mon-d", DaemonName: "d"},
|
|
}
|
|
deployments = append(deployments, apps.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon-c",
|
|
Labels: map[string]string{"ceph_daemon_id": "c"},
|
|
},
|
|
})
|
|
deployments = append(deployments, apps.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon-d",
|
|
Labels: map[string]string{"ceph_daemon_id": "d"},
|
|
},
|
|
})
|
|
c.spec.Mon.Count = 3
|
|
removed = c.checkForExtraMonResources(mons, deployments)
|
|
assert.Equal(t, "", removed)
|
|
}
|
|
|
|
func TestStretchMonVolumeClaimTemplate(t *testing.T) {
|
|
generalSC := "generalSC"
|
|
zoneSC := "zoneSC"
|
|
defaultTemplate := &cephv1.VolumeClaimTemplate{Spec: v1.PersistentVolumeClaimSpec{StorageClassName: &generalSC}}
|
|
zoneTemplate := &cephv1.VolumeClaimTemplate{Spec: v1.PersistentVolumeClaimSpec{StorageClassName: &zoneSC}}
|
|
type fields struct {
|
|
spec cephv1.ClusterSpec
|
|
}
|
|
type args struct {
|
|
mon *monConfig
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
args args
|
|
want *v1.PersistentVolumeClaim
|
|
}{
|
|
{"no template", fields{cephv1.ClusterSpec{}}, args{&monConfig{Zone: "z1"}}, nil},
|
|
{"default template", fields{cephv1.ClusterSpec{Mon: cephv1.MonSpec{VolumeClaimTemplate: defaultTemplate}}}, args{&monConfig{Zone: "z1"}}, defaultTemplate.ToPVC()},
|
|
{
|
|
"default template with 3 zones",
|
|
fields{cephv1.ClusterSpec{Mon: cephv1.MonSpec{
|
|
VolumeClaimTemplate: defaultTemplate,
|
|
StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.MonZoneSpec{{Name: "z1"}, {Name: "z2"}, {Name: "z3"}}},
|
|
}}},
|
|
args{&monConfig{Zone: "z1"}},
|
|
defaultTemplate.ToPVC(),
|
|
},
|
|
{
|
|
"overridden template",
|
|
fields{cephv1.ClusterSpec{Mon: cephv1.MonSpec{
|
|
VolumeClaimTemplate: defaultTemplate,
|
|
StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.MonZoneSpec{{Name: "z1", VolumeClaimTemplate: zoneTemplate}, {Name: "z2"}, {Name: "z3"}}},
|
|
}}},
|
|
args{&monConfig{Zone: "z1"}},
|
|
zoneTemplate.ToPVC(),
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
c := &Cluster{
|
|
spec: tt.fields.spec,
|
|
}
|
|
if got := c.monVolumeClaimTemplate(tt.args.mon); !reflect.DeepEqual(got, tt.want) {
|
|
t.Errorf("Cluster.monVolumeClaimTemplate() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArbiterPlacement(t *testing.T) {
|
|
placement := cephv1.Placement{
|
|
NodeAffinity: &v1.NodeAffinity{
|
|
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
|
|
NodeSelectorTerms: []v1.NodeSelectorTerm{
|
|
{
|
|
MatchExpressions: []v1.NodeSelectorRequirement{
|
|
{
|
|
Key: "foo",
|
|
Operator: v1.NodeSelectorOpExists,
|
|
Values: []string{"bar"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
c := &Cluster{spec: cephv1.ClusterSpec{
|
|
Mon: cephv1.MonSpec{
|
|
StretchCluster: &cephv1.StretchClusterSpec{
|
|
Zones: []cephv1.MonZoneSpec{
|
|
{Name: "a", Arbiter: true},
|
|
{Name: "b"},
|
|
{Name: "c"},
|
|
},
|
|
},
|
|
},
|
|
}}
|
|
|
|
c.spec.Placement = cephv1.PlacementSpec{}
|
|
c.spec.Placement[cephv1.KeyMonArbiter] = placement
|
|
|
|
// No placement is found if not requesting the arbiter placement
|
|
result := c.getMonPlacement("c")
|
|
assert.Equal(t, cephv1.Placement{}, result)
|
|
|
|
// Placement is found if requesting the arbiter
|
|
result = c.getMonPlacement("a")
|
|
assert.Equal(t, placement, result)
|
|
|
|
// Arbiter and all mons have the same placement if no arbiter placement is specified
|
|
c.spec.Placement = cephv1.PlacementSpec{}
|
|
c.spec.Placement[cephv1.KeyMon] = placement
|
|
result = c.getMonPlacement("a")
|
|
assert.Equal(t, placement, result)
|
|
result = c.getMonPlacement("c")
|
|
assert.Equal(t, placement, result)
|
|
}
|
|
|
|
func TestCheckIfArbiterReady(t *testing.T) {
|
|
c := &Cluster{
|
|
Namespace: "ns",
|
|
spec: cephv1.ClusterSpec{
|
|
Mon: cephv1.MonSpec{
|
|
StretchCluster: &cephv1.StretchClusterSpec{
|
|
Zones: []cephv1.MonZoneSpec{
|
|
{Name: "a", Arbiter: true},
|
|
{Name: "b"},
|
|
{Name: "c"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
crushZoneCount := 0
|
|
zone1WeightOverride := 0 // when non-zero, override zone 1's weight (i%2==1)
|
|
balanced := true
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
switch {
|
|
case args[0] == "osd" && args[1] == "crush" && args[2] == "dump":
|
|
crushBuckets := `
|
|
{"id": -1,"name": "default","type_id": 10,"type_name": "root","weight": 1028},
|
|
{"id": -2,"name": "default~hdd","type_id": 10,"type_name": "root","weight": 1028},
|
|
{"id": -3,"name": "mynode","type_id": 1,"type_name": "host","weight": 1028},
|
|
{"id": -4,"name": "mynode~hdd","type_id": 1,"type_name": "host","weight": 1028}`
|
|
for i := 0; i < crushZoneCount; i++ {
|
|
weight := 2056
|
|
if !balanced && i%2 == 1 {
|
|
// simulate unbalanced with every other zone having half the weight
|
|
weight = 1028
|
|
}
|
|
if zone1WeightOverride != 0 && i%2 == 1 {
|
|
weight = zone1WeightOverride
|
|
}
|
|
crushBuckets = crushBuckets +
|
|
fmt.Sprintf(`,{"id": -%d,"name": "zone%d","type_id": 1,"type_name": "zone","weight": %d}
|
|
,{"id": -%d,"name": "zone%d~ssd","type_id": 1,"type_name": "zone","weight": 2056}`, i+5, i, weight, i+6, i)
|
|
}
|
|
return fmt.Sprintf(`{"buckets": [%s]}`, crushBuckets), nil
|
|
}
|
|
return "", fmt.Errorf("unrecognized output file command: %s %v", command, args)
|
|
},
|
|
}
|
|
c.context = &clusterd.Context{Clientset: test.New(t, 5), Executor: executor}
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(5)
|
|
|
|
// Not ready if no pods running
|
|
ready, err := c.readyToConfigureArbiter(true)
|
|
assert.False(t, ready)
|
|
assert.NoError(t, err)
|
|
|
|
// For the remainder of tests, skip checking OSD pods
|
|
// Now there are not enough zones
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.False(t, ready)
|
|
assert.NoError(t, err)
|
|
|
|
// Valid
|
|
crushZoneCount = 2
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.True(t, ready)
|
|
assert.NoError(t, err)
|
|
|
|
// Valid, except the CRUSH map is not balanced (50% difference)
|
|
balanced = false
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.False(t, ready)
|
|
assert.NoError(t, err)
|
|
|
|
// Slightly unbalanced within 10%, but rejected on Ceph without the relaxed check
|
|
balanced = true
|
|
zone1WeightOverride = 2047 // ~0.4% difference from 2056
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.False(t, ready)
|
|
assert.NoError(t, err)
|
|
|
|
// Still rejected on tentacle v20.2.2 (pre-backport)
|
|
c.ClusterInfo.CephVersion = cephver.CephVersion{Major: 20, Minor: 2, Extra: 1}
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.False(t, ready)
|
|
assert.NoError(t, err)
|
|
|
|
// Accepted on tentacle v20.2.2 (ceph/ceph#67790)
|
|
c.ClusterInfo.CephVersion = cephver.CephVersion{Major: 20, Minor: 2, Extra: 2}
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.True(t, ready)
|
|
assert.NoError(t, err)
|
|
|
|
// Accepted on Umbrella+ (ceph/ceph#66580)
|
|
c.ClusterInfo.CephVersion = cephver.Umbrella
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.True(t, ready)
|
|
assert.NoError(t, err)
|
|
|
|
// Over 10% unbalanced, rejected even on supported versions
|
|
zone1WeightOverride = 1850 // ~10.0% difference from 2056
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.False(t, ready)
|
|
assert.NoError(t, err)
|
|
zone1WeightOverride = 0
|
|
|
|
// Too many zones in the CRUSH map
|
|
crushZoneCount = 3
|
|
balanced = true
|
|
ready, err = c.readyToConfigureArbiter(false)
|
|
assert.False(t, ready)
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestCrushWeightsBalanced(t *testing.T) {
|
|
// Equal weights
|
|
assert.True(t, crushWeightsBalanced(2056, 2056))
|
|
|
|
// Within 10%
|
|
assert.True(t, crushWeightsBalanced(2056, 1900)) // ~7.6% diff
|
|
assert.True(t, crushWeightsBalanced(2747124, 2747111)) // from the issue report
|
|
|
|
// Exactly 10%
|
|
assert.True(t, crushWeightsBalanced(1000, 900))
|
|
|
|
// Beyond 10%
|
|
assert.False(t, crushWeightsBalanced(2056, 1028)) // 50% diff
|
|
assert.False(t, crushWeightsBalanced(1000, 899))
|
|
|
|
// Zero weights
|
|
assert.True(t, crushWeightsBalanced(0, 0))
|
|
assert.False(t, crushWeightsBalanced(100, 0))
|
|
}
|
|
|
|
func TestSkipReconcile(t *testing.T) {
|
|
c := New(context.TODO(), &clusterd.Context{Clientset: test.New(t, 1), ConfigDir: t.TempDir()}, "ns", cephv1.ClusterSpec{}, cephclient.NewMinimumOwnerInfoWithOwnerRef())
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
c.ClusterInfo.Namespace = "ns"
|
|
|
|
monDeployment := &apps.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon-a",
|
|
Namespace: c.ClusterInfo.Namespace,
|
|
Labels: map[string]string{
|
|
k8sutil.AppAttr: AppName,
|
|
config.MonType: "a",
|
|
},
|
|
},
|
|
}
|
|
|
|
deployment, err := c.context.Clientset.AppsV1().Deployments(c.ClusterInfo.Namespace).Create(c.ClusterInfo.Context, monDeployment, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
result, err := opcontroller.GetDaemonsToSkipReconcile(c.ClusterInfo.Context, c.context, c.ClusterInfo.Namespace, config.MonType, AppName)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 0, result.Len())
|
|
|
|
deployment, err = c.context.Clientset.AppsV1().Deployments(c.ClusterInfo.Namespace).Get(c.ClusterInfo.Context, deployment.Name, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
deployment.Labels[cephv1.SkipReconcileLabelKey] = ""
|
|
_, err = c.context.Clientset.AppsV1().Deployments(c.ClusterInfo.Namespace).Update(c.ClusterInfo.Context, deployment, metav1.UpdateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
result, err = opcontroller.GetDaemonsToSkipReconcile(c.ClusterInfo.Context, c.context, c.ClusterInfo.Namespace, config.MonType, AppName)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 1, result.Len())
|
|
}
|
|
|
|
func TestHasMonPathChanged(t *testing.T) {
|
|
monDeployment := &apps.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon-a",
|
|
Namespace: "test",
|
|
Labels: map[string]string{
|
|
"pvc_name": "test-pvc",
|
|
},
|
|
},
|
|
}
|
|
|
|
pvcTemplate := &v1.PersistentVolumeClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-claim",
|
|
},
|
|
}
|
|
t.Run("mon path changed from pv to hostpath", func(t *testing.T) {
|
|
assert.True(t, hasMonPathChanged(monDeployment, nil))
|
|
})
|
|
|
|
t.Run("mon path not changed from pv to hostpath", func(t *testing.T) {
|
|
assert.False(t, hasMonPathChanged(monDeployment, pvcTemplate))
|
|
})
|
|
t.Run("mon path changed from hostPath to pvc", func(t *testing.T) {
|
|
delete(monDeployment.Labels, "pvc_name")
|
|
assert.True(t, hasMonPathChanged(monDeployment, pvcTemplate))
|
|
})
|
|
|
|
t.Run("mon path not changed from hostPath to pvc", func(t *testing.T) {
|
|
delete(monDeployment.Labels, "pvc_name")
|
|
assert.False(t, hasMonPathChanged(monDeployment, nil))
|
|
})
|
|
}
|
|
|
|
func TestIsMonIPUpdateRequiredForHostNetwork(t *testing.T) {
|
|
t.Run("both cluster and mon are set to use host network", func(t *testing.T) {
|
|
hostNetwork := &cephv1.NetworkSpec{HostNetwork: true}
|
|
monUsingHostNetwork := true
|
|
assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
|
|
})
|
|
|
|
t.Run("both cluster and mon are not set for host network", func(t *testing.T) {
|
|
hostNetwork := &cephv1.NetworkSpec{}
|
|
monUsingHostNetwork := false
|
|
assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
|
|
})
|
|
t.Run("cluster is set for host networking but mon pod is not", func(t *testing.T) {
|
|
hostNetwork := &cephv1.NetworkSpec{HostNetwork: true}
|
|
monUsingHostNetwork := false
|
|
assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
|
|
})
|
|
|
|
t.Run("mon is using host networking but cluster is updated to not use host network ", func(t *testing.T) {
|
|
hostNetwork := &cephv1.NetworkSpec{}
|
|
monUsingHostNetwork := true
|
|
assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
|
|
})
|
|
|
|
t.Run("mon is using host networking and cluster is set host network via NetworkProviderHost ", func(t *testing.T) {
|
|
hostNetwork := &cephv1.NetworkSpec{Provider: cephv1.NetworkProviderHost}
|
|
monUsingHostNetwork := true
|
|
assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
|
|
})
|
|
|
|
t.Run("mon is not using host networking but cluster is updated to use host network via NetworkProviderHost ", func(t *testing.T) {
|
|
hostNetwork := &cephv1.NetworkSpec{Provider: cephv1.NetworkProviderHost}
|
|
monUsingHostNetwork := false
|
|
assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
|
|
})
|
|
}
|
|
|
|
func TestRotateMonCephxKeys(t *testing.T) {
|
|
ctx := context.TODO()
|
|
namespace := "default"
|
|
context, err := newTestStartCluster(t, namespace)
|
|
assert.NoError(t, err)
|
|
c := newCluster(context, namespace, true, v1.ResourceRequirements{})
|
|
|
|
uninitializedStatus := keyring.UninitializedCephxStatus()
|
|
cluster := &cephv1.CephCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: namespace,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: cephv1.ClusterSpec{
|
|
Security: cephv1.ClusterSecuritySpec{
|
|
CephX: cephv1.ClusterCephxConfig{
|
|
Daemon: cephv1.CephxConfig{},
|
|
},
|
|
},
|
|
},
|
|
Status: cephv1.ClusterStatus{
|
|
Cephx: cephv1.ClusterCephxStatus{
|
|
Mon: uninitializedStatus,
|
|
},
|
|
},
|
|
}
|
|
|
|
s := scheme.Scheme
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
|
|
s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{})
|
|
assert.NoError(t, csiopv1.AddToScheme(s))
|
|
|
|
object := []runtime.Object{
|
|
cluster,
|
|
}
|
|
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
|
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
logger.Infof("%s %v", command, args)
|
|
if args[0] == "auth" && args[1] == "rotate" {
|
|
return `[{"key":"myrotatedkey"}]`, nil
|
|
}
|
|
return "", errors.New("unknown command")
|
|
},
|
|
}
|
|
c.context = &clusterd.Context{Clientset: test.New(t, 5), Executor: executor, Client: cl}
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
c.ClusterInfo.CephVersion = keyring.CephAuthMonRotateSupportedVersion
|
|
|
|
// create rook-ceph-mon secret
|
|
secret := &v1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: AppName,
|
|
Namespace: namespace,
|
|
},
|
|
Data: map[string][]byte{opcontroller.MonSecretNameKey: []byte("bar")},
|
|
}
|
|
|
|
_, err = c.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
// No key rotation required
|
|
shouldRotate, err := c.RotateMonCephxKeys(cluster)
|
|
assert.NoError(t, err)
|
|
assert.False(t, shouldRotate)
|
|
|
|
// verify key is rotated in the secret
|
|
cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy
|
|
cluster.Spec.Security.CephX.Daemon.KeyGeneration = 2
|
|
cluster.Status.Cephx.Mon.KeyGeneration = 1
|
|
cluster.Status.Cephx.Mon.KeyCephVersion = "19.2.3-0"
|
|
err = c.context.Client.Update(ctx, cluster)
|
|
assert.NoError(t, err)
|
|
shouldRotate, err = c.RotateMonCephxKeys(cluster)
|
|
assert.NoError(t, err)
|
|
assert.True(t, shouldRotate)
|
|
secret, err = c.context.Clientset.CoreV1().Secrets(namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "myrotatedkey", string(secret.Data[opcontroller.MonSecretNameKey]))
|
|
|
|
// verify mon key does not rotate when other daemons rotate
|
|
cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy
|
|
cluster.Spec.Security.CephX.Daemon.KeyGeneration = 2
|
|
cluster.Status.Cephx.Mon.KeyGeneration = 1
|
|
cluster.Status.Cephx.Mon.KeyCephVersion = "19.2.3-0"
|
|
err = c.context.Client.Update(ctx, cluster)
|
|
assert.NoError(t, err)
|
|
c.ClusterInfo.CephVersion = keyring.CephAuthRotateSupportedVersion // but not CephAuthMonRotateSupportedVersion
|
|
shouldRotate, err = c.RotateMonCephxKeys(cluster)
|
|
assert.NoError(t, err)
|
|
assert.False(t, shouldRotate)
|
|
}
|
|
|
|
func newFloatingMonTestCluster(t *testing.T, namespace string) *Cluster {
|
|
t.Helper()
|
|
ctx, err := newTestStartCluster(t, namespace)
|
|
assert.NoError(t, err)
|
|
c := newCluster(ctx, namespace, true, v1.ResourceRequirements{})
|
|
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
|
c.ClusterInfo.CephVersion = cephver.Squid
|
|
c.spec.CephVersion = cephv1.CephVersionSpec{Image: "quay.io/ceph/ceph:v18"}
|
|
c.spec.DataDirHostPath = "/var/lib/rook"
|
|
c.ClusterInfo.FSID = "test-fsid"
|
|
|
|
_, err = c.context.Clientset.CoreV1().ConfigMaps(namespace).Create(c.ClusterInfo.Context, &v1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: EndpointConfigMapName},
|
|
Data: map[string]string{"maxMonId": "2"},
|
|
}, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
_, err = c.context.Clientset.CoreV1().ConfigMaps(namespace).Create(c.ClusterInfo.Context, &v1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "floating-vars", Namespace: namespace},
|
|
Data: map[string]string{
|
|
"DRBD_CONF_PATH": "/etc/drbd.d/mon.res", "DRBD_DIR_PATH": "/var/lib/drbd",
|
|
"DRBD_DEVICE_NAME": "/dev/drbd100", "DRBD_UTILS_IMAGE": "drbd-utils:latest",
|
|
},
|
|
}, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
c.spec.Mon.FloatingMon = cephv1.FloatingMonSpec{Name: "c", ConfigMapName: "floating-vars"}
|
|
return c
|
|
}
|
|
|
|
func TestFloatingMonParamValidation(t *testing.T) {
|
|
namespace := "ns"
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
monCfg := &monConfig{
|
|
DaemonName: "c", PublicIP: "10.0.0.100", Port: DefaultMsgr1Port,
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("c"), config.MonType, "c", namespace),
|
|
}
|
|
|
|
// built-in and configmap params are populated
|
|
t.Run("configmap merges with built-in params", func(t *testing.T) {
|
|
p, err := c.buildFloatingMonTemplateParams(c.spec.Mon.FloatingMon, monCfg, monCfg.DataPathMap, c.spec.DataDirHostPath)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "c", p["NAME"])
|
|
assert.Equal(t, namespace, p["NAMESPACE"])
|
|
assert.Equal(t, "test-fsid", p["FSID"])
|
|
assert.Equal(t, "quay.io/ceph/ceph:v18", p["CEPH_IMAGE"])
|
|
assert.Equal(t, "10.0.0.100", p["PUBLIC_IP"])
|
|
assert.Equal(t, "/etc/drbd.d/mon.res", p["DRBD_CONF_PATH"])
|
|
assert.Equal(t, "/dev/drbd100", p["DRBD_DEVICE_NAME"])
|
|
})
|
|
|
|
// referencing a configmap that does not exist must fail
|
|
t.Run("missing configmap errors", func(t *testing.T) {
|
|
spec := cephv1.FloatingMonSpec{Name: "c", ConfigMapName: "does-not-exist"}
|
|
_, err := c.buildFloatingMonTemplateParams(spec, monCfg, monCfg.DataPathMap, c.spec.DataDirHostPath)
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "failed to get floating mon configmap")
|
|
})
|
|
|
|
// configmap with empty data should still produce valid built-in params
|
|
t.Run("empty configmap data still has built-ins", func(t *testing.T) {
|
|
_, err := c.context.Clientset.CoreV1().ConfigMaps(namespace).Create(c.ClusterInfo.Context, &v1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "empty-cm", Namespace: namespace},
|
|
Data: map[string]string{},
|
|
}, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
spec := cephv1.FloatingMonSpec{Name: "c", ConfigMapName: "empty-cm"}
|
|
p, err := c.buildFloatingMonTemplateParams(spec, monCfg, monCfg.DataPathMap, c.spec.DataDirHostPath)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "c", p["NAME"])
|
|
assert.Equal(t, "quay.io/ceph/ceph:v18", p["CEPH_IMAGE"])
|
|
assert.Empty(t, p["DRBD_CONF_PATH"])
|
|
})
|
|
|
|
// configmap keys that collide with built-ins must not override them
|
|
t.Run("configmap cannot override built-in params", func(t *testing.T) {
|
|
_, err := c.context.Clientset.CoreV1().ConfigMaps(namespace).Create(c.ClusterInfo.Context, &v1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "bad-cm", Namespace: namespace},
|
|
Data: map[string]string{"NAME": "hijacked", "FSID": "fake-fsid", "CUSTOM_KEY": "custom-val"},
|
|
}, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
spec := cephv1.FloatingMonSpec{Name: "c", ConfigMapName: "bad-cm"}
|
|
p, err := c.buildFloatingMonTemplateParams(spec, monCfg, monCfg.DataPathMap, c.spec.DataDirHostPath)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "c", p["NAME"], "built-in NAME must not be overridden by configmap")
|
|
assert.Equal(t, "test-fsid", p["FSID"], "built-in FSID must not be overridden by configmap")
|
|
assert.Equal(t, "custom-val", p["CUSTOM_KEY"], "non-colliding configmap keys should pass through")
|
|
})
|
|
}
|
|
|
|
func TestFloatingMonDeployment(t *testing.T) {
|
|
namespace := "ns"
|
|
m := &monConfig{
|
|
ResourceName: "rook-ceph-mon-c", DaemonName: "c",
|
|
Port: DefaultMsgr1Port, PublicIP: "10.0.0.100",
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("c"), config.MonType, "c", namespace),
|
|
}
|
|
|
|
// basic deployment has a mon container with the right image and init containers
|
|
t.Run("creates deployment with mon container", func(t *testing.T) {
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
assert.NoError(t, c.startMon(m, nil))
|
|
|
|
d, err := c.context.Clientset.AppsV1().Deployments(namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "rook-ceph-mon-c", d.Name)
|
|
assert.NotEmpty(t, d.Spec.Template.Spec.InitContainers)
|
|
|
|
foundMon := false
|
|
for _, ct := range d.Spec.Template.Spec.Containers {
|
|
if ct.Name == "mon" {
|
|
foundMon = true
|
|
assert.Equal(t, "quay.io/ceph/ceph:v18", ct.Image)
|
|
}
|
|
}
|
|
assert.True(t, foundMon, "expected mon container")
|
|
})
|
|
|
|
// resource limits are applied to the mon container
|
|
t.Run("resource limits applied", func(t *testing.T) {
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
c.spec.Resources = map[string]v1.ResourceRequirements{
|
|
cephv1.ResourcesKeyMon: {Limits: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("500m"),
|
|
v1.ResourceMemory: resource.MustParse("2Gi"),
|
|
}},
|
|
cephv1.ResourcesKeyLogCollector: {Limits: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("10m"),
|
|
v1.ResourceMemory: resource.MustParse("32Mi"),
|
|
}},
|
|
cephv1.ResourcesKeyFloatingMonShutDownApp: {Limits: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("20m"),
|
|
v1.ResourceMemory: resource.MustParse("48Mi"),
|
|
}},
|
|
}
|
|
assert.NoError(t, c.startMon(m, nil))
|
|
|
|
d, err := c.context.Clientset.AppsV1().Deployments(namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
for _, ct := range d.Spec.Template.Spec.Containers {
|
|
if ct.Name == cephv1.ResourcesKeyMon {
|
|
assert.Equal(t, "500m", ct.Resources.Limits.Cpu().String())
|
|
assert.Equal(t, "2Gi", ct.Resources.Limits.Memory().String())
|
|
}
|
|
if strings.Contains(ct.Name, "log") {
|
|
assert.Equal(t, "10m", ct.Resources.Limits.Cpu().String())
|
|
assert.Equal(t, "32Mi", ct.Resources.Limits.Memory().String())
|
|
}
|
|
if ct.Name == cephv1.ResourcesKeyFloatingMonShutDownApp {
|
|
assert.Equal(t, "20m", ct.Resources.Limits.Cpu().String())
|
|
assert.Equal(t, "48Mi", ct.Resources.Limits.Memory().String())
|
|
}
|
|
}
|
|
})
|
|
|
|
// mon annotations and labels from the CRD are applied
|
|
t.Run("annotations and labels applied", func(t *testing.T) {
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
c.spec.Annotations = cephv1.AnnotationsSpec{cephv1.KeyMon: cephv1.Annotations{"my-anno": "anno-val"}}
|
|
c.spec.Labels = cephv1.LabelsSpec{cephv1.KeyMon: cephv1.Labels{"my-label": "label-val"}}
|
|
assert.NoError(t, c.startMon(m, nil))
|
|
|
|
d, err := c.context.Clientset.AppsV1().Deployments(namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "anno-val", d.Annotations["my-anno"])
|
|
assert.Equal(t, "label-val", d.Labels["my-label"])
|
|
})
|
|
|
|
t.Run("update path on existing deployment", func(t *testing.T) {
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
assert.NoError(t, c.startMon(m, nil))
|
|
|
|
updateCalled := false
|
|
updateDeploymentAndWait = func(_ *clusterd.Context, _ *cephclient.ClusterInfo, _ *apps.Deployment, _, _ string, _, _ bool) error {
|
|
updateCalled = true
|
|
return nil
|
|
}
|
|
|
|
assert.NoError(t, c.startMon(m, nil))
|
|
assert.True(t, updateCalled)
|
|
})
|
|
|
|
// PVC volumes are not injected even when VolumeClaimTemplate is set
|
|
t.Run("no PVC volumes injected", func(t *testing.T) {
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
c.spec.Mon.VolumeClaimTemplate = &cephv1.VolumeClaimTemplate{}
|
|
assert.NoError(t, c.startMon(m, nil))
|
|
|
|
d, err := c.context.Clientset.AppsV1().Deployments(namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
for _, vol := range d.Spec.Template.Spec.Volumes {
|
|
assert.Nil(t, vol.PersistentVolumeClaim)
|
|
}
|
|
})
|
|
|
|
// missing configmap prevents deployment creation
|
|
t.Run("missing configmap fails", func(t *testing.T) {
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
c.spec.Mon.FloatingMon = cephv1.FloatingMonSpec{Name: "c", ConfigMapName: "nonexistent"}
|
|
|
|
err := c.startMon(m, nil)
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "failed to get floating mon configmap")
|
|
})
|
|
}
|
|
|
|
func TestFloatingMonLabelsAndSchedulingSkip(t *testing.T) {
|
|
namespace := "ns"
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
|
|
floatingM := &monConfig{
|
|
ResourceName: "rook-ceph-mon-c", DaemonName: "c",
|
|
Port: DefaultMsgr1Port, PublicIP: "10.0.0.100",
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("c"), config.MonType, "c", namespace),
|
|
}
|
|
assert.NoError(t, c.startMon(floatingM, nil))
|
|
|
|
normalM := &monConfig{
|
|
ResourceName: "rook-ceph-mon-a", DaemonName: "a",
|
|
Port: DefaultMsgr1Port, PublicIP: "1.2.3.4",
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("a"), config.MonType, "a", namespace),
|
|
}
|
|
assert.NoError(t, c.startMon(normalM, &opcontroller.MonScheduleInfo{Hostname: "host-a"}))
|
|
|
|
floatingDeploy, err := c.context.Clientset.AppsV1().Deployments(namespace).Get(c.ClusterInfo.Context, floatingM.ResourceName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
normalDeploy, err := c.context.Clientset.AppsV1().Deployments(namespace).Get(c.ClusterInfo.Context, normalM.ResourceName, metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
// labels differ between floating and normal mons
|
|
assert.Equal(t, FloatingMonAppName, floatingDeploy.Labels["app"])
|
|
assert.Equal(t, FloatingMonAppName, floatingDeploy.Spec.Selector.MatchLabels["app"])
|
|
assert.Equal(t, AppName, normalDeploy.Labels["app"])
|
|
|
|
// assignMons must skip the floating mon during canary scheduling
|
|
var scheduled []string
|
|
waitForMonitorScheduling = func(c *Cluster, d *apps.Deployment) (SchedulingResult, error) {
|
|
scheduled = append(scheduled, d.Labels[opcontroller.DaemonIDLabel])
|
|
node, _ := c.context.Clientset.CoreV1().Nodes().Get(c.ClusterInfo.Context, "node0", metav1.GetOptions{})
|
|
return SchedulingResult{Node: node}, nil
|
|
}
|
|
|
|
mons := []*monConfig{
|
|
{
|
|
ResourceName: "rook-ceph-mon-a", DaemonName: "a", Port: DefaultMsgr1Port,
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("a"), config.MonType, "a", namespace),
|
|
},
|
|
{
|
|
ResourceName: "rook-ceph-mon-b", DaemonName: "b", Port: DefaultMsgr1Port,
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("b"), config.MonType, "b", namespace),
|
|
},
|
|
{
|
|
ResourceName: "rook-ceph-mon-c", DaemonName: "c", Port: DefaultMsgr1Port,
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("c"), config.MonType, "c", namespace),
|
|
},
|
|
}
|
|
|
|
assert.NoError(t, c.assignMons(mons, sets.New[string]()))
|
|
assert.Contains(t, scheduled, "a")
|
|
assert.Contains(t, scheduled, "b")
|
|
assert.NotContains(t, scheduled, "c")
|
|
}
|
|
|
|
// TestAssignMonsConcurrentSchedulingFailure ensures assignMons does not race on
|
|
// the shared failedMonSchedule flag when multiple mons fail to schedule at the
|
|
// same time. Each mon is scheduled in its own goroutine, so several goroutines
|
|
// can set failedMonSchedule concurrently. Run with -race to catch the write-write
|
|
// data race.
|
|
func TestAssignMonsConcurrentSchedulingFailure(t *testing.T) {
|
|
namespace := "ns"
|
|
c := newFloatingMonTestCluster(t, namespace)
|
|
|
|
origWaitForMonitorScheduling := waitForMonitorScheduling
|
|
defer func() { waitForMonitorScheduling = origWaitForMonitorScheduling }()
|
|
|
|
mons := []*monConfig{
|
|
{
|
|
ResourceName: "rook-ceph-mon-a", DaemonName: "a", Port: DefaultMsgr1Port,
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("a"), config.MonType, "a", namespace),
|
|
},
|
|
{
|
|
ResourceName: "rook-ceph-mon-b", DaemonName: "b", Port: DefaultMsgr1Port,
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("b"), config.MonType, "b", namespace),
|
|
},
|
|
{
|
|
ResourceName: "rook-ceph-mon-d", DaemonName: "d", Port: DefaultMsgr1Port,
|
|
DataPathMap: config.NewStatefulDaemonDataPathMap("/var/lib/rook", dataDirRelativeHostPath("d"), config.MonType, "d", namespace),
|
|
},
|
|
}
|
|
|
|
// release all scheduling goroutines at the same time so their concurrent
|
|
// writes to failedMonSchedule overlap, making the data race detectable
|
|
var ready sync.WaitGroup
|
|
ready.Add(len(mons))
|
|
start := make(chan struct{})
|
|
waitForMonitorScheduling = func(c *Cluster, d *apps.Deployment) (SchedulingResult, error) {
|
|
ready.Done()
|
|
<-start
|
|
return SchedulingResult{}, errors.New("mock scheduling failure")
|
|
}
|
|
go func() {
|
|
ready.Wait()
|
|
close(start)
|
|
}()
|
|
|
|
err := c.assignMons(mons, sets.New[string]())
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "failed to schedule mons")
|
|
}
|