Files
my-rook-config/pkg/cephmgr/mon/leader_test.go
T

267 lines
8.9 KiB
Go

/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mon
import (
"fmt"
"strings"
"testing"
"time"
"github.com/rook/rook/pkg/cephmgr/client/test"
cephtest "github.com/rook/rook/pkg/cephmgr/test"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
"github.com/rook/rook/pkg/util"
"github.com/stretchr/testify/assert"
)
func TestMonSelection(t *testing.T) {
etcdClient := util.NewMockEtcdClient()
nodes := make(map[string]*inventory.NodeConfig)
inv := &inventory.Config{Nodes: nodes}
nodes["a"] = &inventory.NodeConfig{PublicIP: "1.2.3.4"}
context := &clusterd.Context{
EtcdClient: etcdClient,
Inventory: inv,
ConfigDir: "/tmp",
}
// choose 1 mon from the set of 1 nodes
chosen, bad, err := chooseMonitorNodes(context)
assert.Nil(t, err)
assert.Equal(t, 1, len(chosen))
assert.Equal(t, 0, len(bad))
// no new monitors when we have 2 nodes
nodes["b"] = &inventory.NodeConfig{PublicIP: "2.2.3.4"}
chosen, bad, err = chooseMonitorNodes(context)
assert.Nil(t, err)
assert.Equal(t, 1, len(chosen))
assert.Equal(t, 0, len(bad))
assert.Equal(t, "mon0", chosen["a"].Name)
// add two more monitors when we hit the threshold of 3 nodes
nodes["c"] = &inventory.NodeConfig{PublicIP: "3.2.3.4"}
chosen, bad, err = chooseMonitorNodes(context)
assert.Nil(t, err)
assert.Equal(t, 3, len(chosen))
assert.Equal(t, 0, len(bad))
assert.Equal(t, "mon0", chosen["a"].Name)
assert.True(t, chosen["b"].Name == "mon1" || chosen["b"].Name == "mon2")
assert.True(t, chosen["c"].Name == "mon1" || chosen["c"].Name == "mon2")
assert.NotEqual(t, chosen["b"].Name, chosen["c"].Name)
// remove a node, then check that with four nodes we only go to three monitors
etcdClient.DeleteDir("/rook/services/ceph/monitor/desired/c")
nodes["c"] = &inventory.NodeConfig{PublicIP: "4.2.3.4"}
nodes["d"] = &inventory.NodeConfig{PublicIP: "5.2.3.4"}
chosen, bad, err = chooseMonitorNodes(context)
assert.Nil(t, err)
assert.Equal(t, 0, len(bad))
assert.Equal(t, 3, len(chosen))
assert.Equal(t, "mon0", chosen["a"].Name)
}
func TestMonitorCount(t *testing.T) {
assert.Equal(t, 0, calculateMonitorCount(0))
assert.Equal(t, 1, calculateMonitorCount(1))
assert.Equal(t, 1, calculateMonitorCount(2))
assert.Equal(t, 3, calculateMonitorCount(3))
assert.Equal(t, 3, calculateMonitorCount(4))
assert.Equal(t, 3, calculateMonitorCount(20))
assert.Equal(t, 5, calculateMonitorCount(21))
assert.Equal(t, 5, calculateMonitorCount(100))
assert.Equal(t, 7, calculateMonitorCount(101))
assert.Equal(t, 7, calculateMonitorCount(1001))
}
func TestMonOnUnhealthyNode(t *testing.T) {
etcdClient := util.NewMockEtcdClient()
// mock a monitor
cephtest.CreateClusterInfo(etcdClient, []string{"a"})
// the monitor is on the bad node
badNode := &clusterd.UnhealthyNode{ID: "a"}
context := &clusterd.Context{EtcdClient: etcdClient, ConfigDir: "/tmp"}
response, err := monsOnUnhealthyNode(context, []*clusterd.UnhealthyNode{badNode})
assert.True(t, response)
assert.Nil(t, err)
// the monitor is not on another node
badNode.ID = "b"
response, err = monsOnUnhealthyNode(context, []*clusterd.UnhealthyNode{badNode})
assert.False(t, response)
assert.Nil(t, err)
}
func TestMoveUnhealthyMonitor(t *testing.T) {
factory := &test.MockConnectionFactory{Fsid: "myfsid", SecretKey: "mykey"}
etcdClient := util.NewMockEtcdClient()
waitForQuorum := func(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error {
return nil
}
leader := &Leader{waitForQuorum: waitForQuorum}
nodes := make(map[string]*inventory.NodeConfig)
inv := &inventory.Config{Nodes: nodes}
nodes["a"] = &inventory.NodeConfig{PublicIP: "1.2.3.4"}
nodes["b"] = &inventory.NodeConfig{PublicIP: "2.3.4.5"}
nodes["c"] = &inventory.NodeConfig{PublicIP: "3.4.5.6"}
context := &clusterd.Context{
EtcdClient: etcdClient,
Inventory: inv,
ConfigDir: "/tmp",
}
// mock the agent responses that the deployments were successful to start mons and osds
etcdClient.WatcherResponses["/rook/_notify/a/monitor/status"] = "succeeded"
etcdClient.WatcherResponses["/rook/_notify/b/monitor/status"] = "succeeded"
etcdClient.WatcherResponses["/rook/_notify/c/monitor/status"] = "succeeded"
// initialize the first three monitors
err := leader.Configure(context, factory, "mykey")
assert.Nil(t, err)
assert.True(t, etcdClient.GetChildDirs("/rook/services/ceph/monitor/desired").Equals(util.CreateSet([]string{"a", "b", "c"})))
// add a new node and mark node a as unhealthy
nodes["a"].HeartbeatAge = (UnhealthyHeartbeatAgeSeconds + 1) * time.Second
nodes["d"] = &inventory.NodeConfig{PublicIP: "4.5.6.7"}
etcdClient.WatcherResponses["/rook/_notify/d/monitor/status"] = "succeeded"
err = leader.Configure(context, factory, "mykey")
assert.Nil(t, err)
assert.True(t, etcdClient.GetChildDirs("/rook/services/ceph/monitor/desired").Equals(util.CreateSet([]string{"d", "b", "c"})))
cluster, err := LoadClusterInfo(etcdClient)
assert.Nil(t, err)
assert.Equal(t, 3, len(cluster.Monitors))
mon1 := false
mon2 := false
mon3 := false
for _, mon := range cluster.Monitors {
if strings.Contains(mon.Endpoint, nodes["a"].PublicIP) {
assert.Fail(t, "mon a was not removed")
}
if strings.Contains(mon.Endpoint, nodes["b"].PublicIP) {
mon1 = true
}
if strings.Contains(mon.Endpoint, nodes["c"].PublicIP) {
mon2 = true
}
if strings.Contains(mon.Endpoint, nodes["d"].PublicIP) {
mon3 = true
}
}
assert.True(t, mon1)
assert.True(t, mon2)
assert.True(t, mon3)
}
func TestMaxMonID(t *testing.T) {
// the empty list returns -1
mons := map[string]*CephMonitorConfig{}
max, err := getMaxMonitorID(mons)
assert.Nil(t, err)
assert.Equal(t, -1, max)
testBadMonID(t, "")
testBadMonID(t, "m")
testBadMonID(t, "m1")
testBadMonID(t, "m100")
testBadMonID(t, "1mon")
testBadMonID(t, "badmon")
testBadMonID(t, "mons1")
testGoodMonID(t, "mon0", 0)
testGoodMonID(t, "mon10", 10)
testGoodMonID(t, "mon123", 123)
}
func testBadMonID(t *testing.T, name string) {
mons := map[string]*CephMonitorConfig{}
mons["a"] = &CephMonitorConfig{Name: name}
_, err := getMaxMonitorID(mons)
assert.NotNil(t, err, fmt.Sprintf("bad mon=%s", name))
}
func testGoodMonID(t *testing.T, name string, expected int) {
mons := map[string]*CephMonitorConfig{}
mons["a"] = &CephMonitorConfig{Name: name}
actual, err := getMaxMonitorID(mons)
assert.Nil(t, err)
assert.Equal(t, expected, actual)
}
func TestUnhealthyMon(t *testing.T) {
etcdClient := util.NewMockEtcdClient()
nodes := make(map[string]*inventory.NodeConfig)
inv := &inventory.Config{Nodes: nodes}
nodes["a"] = &inventory.NodeConfig{PublicIP: "1.2.3.4", HeartbeatAge: UnhealthyHeartbeatAgeSeconds * time.Second}
nodes["b"] = &inventory.NodeConfig{PublicIP: "2.2.3.4"}
nodes["c"] = &inventory.NodeConfig{PublicIP: "3.2.3.4"}
nodes["d"] = &inventory.NodeConfig{PublicIP: "4.2.3.4"}
context := &clusterd.Context{
EtcdClient: etcdClient,
Inventory: inv,
ConfigDir: "/tmp",
}
// choose 3 mons from the set of 4 nodes, but don't choose the unhealthy node 'a'
chosen, bad, err := chooseMonitorNodes(context)
assert.Nil(t, err)
assert.Equal(t, 3, len(chosen))
_, found := chosen["a"]
assert.False(t, found)
bmonName := chosen["b"].Name
cmonName := chosen["c"].Name
desiredMons := etcdClient.GetChildDirs("/rook/services/ceph/monitor/desired")
assert.True(t, desiredMons.Contains("b"))
assert.True(t, desiredMons.Contains("c"))
assert.True(t, desiredMons.Contains("d"))
assert.False(t, desiredMons.Contains("a"))
// now we expect to add a and remove b from the desired state
nodes["a"].HeartbeatAge = 0
nodes["b"].HeartbeatAge = UnhealthyHeartbeatAgeSeconds * time.Second
chosen, bad, err = chooseMonitorNodes(context)
assert.Nil(t, err)
assert.Equal(t, 3, len(chosen))
assert.Equal(t, 1, len(bad))
assert.Equal(t, "mon3", chosen["a"].Name)
assert.Equal(t, bmonName, bad["b"].Name)
desiredMons = etcdClient.GetChildDirs("/rook/services/ceph/monitor/desired")
assert.True(t, desiredMons.Contains("a"))
assert.True(t, desiredMons.Contains("c"))
assert.True(t, desiredMons.Contains("d"))
assert.False(t, desiredMons.Contains("b"))
// fail choosing the nodes if we don't have enough healthy nodes to move.
// 'b' is still unhealthy and 'c' now becomes unhealthy
nodes["c"].HeartbeatAge = UnhealthyHeartbeatAgeSeconds * time.Second
chosen, bad, err = chooseMonitorNodes(context)
assert.NotNil(t, err)
assert.Equal(t, 1, len(bad))
assert.Equal(t, cmonName, bad["c"].Name)
}