handle ceph mon failover more gracefully
Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
5 files changed
+184
-77
No files matched your search
@@ -462,7 +462,7 @@ spec:
|
||||
# The duration to wait before trying to failover or remove/replace the
|
||||
# current mon with a new mon (useful for compensating flapping network).
|
||||
- name: ROOK_MON_OUT_TIMEOUT
|
||||
value: "300s"
|
||||
value: "600s"
|
||||
# The duration between discovering devices in the rook-discover daemonset.
|
||||
- name: ROOK_DISCOVER_DEVICES_INTERVAL
|
||||
value: "60m"
|
||||
|
||||
@@ -134,7 +134,7 @@ func (c *Cluster) checkHealth() error {
|
||||
// when the timeout for the mon has been reached, continue to the
|
||||
// normal failover/delete mon pod part of the code
|
||||
if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout {
|
||||
logger.Warningf("mon %s not found in quorum, still in mon out timeout", mon.Name)
|
||||
logger.Warningf("mon %s not found in quorum, waiting for timeout before failover", mon.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -281,7 +281,7 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
c.clusterInfo.Monitors[m.DaemonName] = cephconfig.NewMonInfo(m.DaemonName, m.PublicIP, m.Port)
|
||||
|
||||
// Start the deployment
|
||||
if err = c.startDeployments(mConf, len(mConf)-1); err != nil {
|
||||
if err = c.startDeployments(mConf, true); err != nil {
|
||||
return fmt.Errorf("failed to start new mon %s. %+v", m.DaemonName, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -176,48 +176,76 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
|
||||
func (c *Cluster) startMons() error {
|
||||
// init the mons config
|
||||
mons := c.initMonConfig(c.Count)
|
||||
// init the mon config
|
||||
existingCount, mons := c.initMonConfig(c.Count)
|
||||
|
||||
// Assign the pods to nodes
|
||||
// Assign the mons to nodes
|
||||
if err := c.assignMons(mons); err != nil {
|
||||
return fmt.Errorf("failed to assign pods to mons. %+v", err)
|
||||
}
|
||||
|
||||
// Start one monitor at a time
|
||||
for i := 0; i < c.Count; i++ {
|
||||
logger.Infof("ensuring mon %s (%s) is started", mons[i].ResourceName, mons[i].DaemonName)
|
||||
endIndex := len(c.clusterInfo.Monitors)
|
||||
if endIndex < c.Count {
|
||||
endIndex++
|
||||
if existingCount < len(mons) {
|
||||
// Start the new mons one at a time
|
||||
for i := existingCount; i < c.Count; i++ {
|
||||
if err := c.ensureMonsRunning(mons, i, true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
logger.Infof("looping to start mons. i=%d, endIndex=%d, c.Size=%d", i, endIndex, c.Count)
|
||||
|
||||
// Init the mon IPs
|
||||
if err := c.initMonIPs(mons[0:endIndex]); err != nil {
|
||||
return fmt.Errorf("failed to init mon services. %+v", err)
|
||||
}
|
||||
|
||||
// save the mon config after we have "initiated the IPs"
|
||||
if err := c.saveMonConfig(); err != nil {
|
||||
return fmt.Errorf("failed to save mons. %+v", err)
|
||||
}
|
||||
|
||||
// make sure we have the connection info generated so connections can happen
|
||||
if err := writeConnectionConfig(c.context, c.clusterInfo); err != nil {
|
||||
} else {
|
||||
// Ensure all the expected mon deployments exist, but don't require full quorum to continue
|
||||
lastMonIndex := len(mons) - 1
|
||||
if err := c.ensureMonsRunning(mons, lastMonIndex, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Start the deployment
|
||||
if err := c.startDeployments(mons[0:endIndex], i); err != nil {
|
||||
return fmt.Errorf("failed to start mon pods. %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("mon endpoints used are: %s", FlattenMonEndpoints(c.clusterInfo.Monitors))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureMonsRunning is called in two scenarios:
|
||||
// 1. To create a new mon and wait for it to join quorum (requireAllInQuorum = true). This method will be called multiple times
|
||||
// to add a mon until we have reached the desired number of mons.
|
||||
// 2. To check that the majority of existing mons are in quorum. It is ok if not all mons are in quorum. (requireAllInQuorum = false)
|
||||
// This is needed when the operator is restarted and all mons may not be up or in quorum.
|
||||
func (c *Cluster) ensureMonsRunning(mons []*monConfig, i int, requireAllInQuorum bool) error {
|
||||
if requireAllInQuorum {
|
||||
logger.Infof("creating mon %s", mons[i].DaemonName)
|
||||
} else {
|
||||
logger.Info("checking for basic quorum with existing mons")
|
||||
}
|
||||
|
||||
// Calculate how many mons we expected to exist after this method is completed.
|
||||
// If we are adding a new mon, we expect one more than currently exist.
|
||||
// If we haven't created all the desired mons already, we will be adding a new one with this iteration
|
||||
expectedMonCount := len(c.clusterInfo.Monitors)
|
||||
if expectedMonCount < c.Count {
|
||||
expectedMonCount++
|
||||
}
|
||||
|
||||
// Init the mon IPs
|
||||
if err := c.initMonIPs(mons[0:expectedMonCount]); err != nil {
|
||||
return fmt.Errorf("failed to init mon services. %+v", err)
|
||||
}
|
||||
|
||||
// save the mon config after we have "initiated the IPs"
|
||||
if err := c.saveMonConfig(); err != nil {
|
||||
return fmt.Errorf("failed to save mons. %+v", err)
|
||||
}
|
||||
|
||||
// make sure we have the connection info generated so connections can happen
|
||||
if err := writeConnectionConfig(c.context, c.clusterInfo); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Start the deployment
|
||||
if err := c.startDeployments(mons[0:expectedMonCount], requireAllInQuorum); err != nil {
|
||||
return fmt.Errorf("failed to start mon pods. %+v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initClusterInfo retrieves the ceph cluster info if it already exists.
|
||||
// If a new cluster, create new keys.
|
||||
func (c *Cluster) initClusterInfo() error {
|
||||
@@ -240,7 +268,7 @@ func (c *Cluster) initClusterInfo() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) initMonConfig(size int) []*monConfig {
|
||||
func (c *Cluster) initMonConfig(size int) (int, []*monConfig) {
|
||||
mons := []*monConfig{}
|
||||
|
||||
// initialize the mon pod info for mons that have been previously created
|
||||
@@ -255,12 +283,13 @@ func (c *Cluster) initMonConfig(size int) []*monConfig {
|
||||
}
|
||||
|
||||
// initialize mon info if we don't have enough mons (at first startup)
|
||||
existingCount := len(c.clusterInfo.Monitors)
|
||||
for i := len(c.clusterInfo.Monitors); i < size; i++ {
|
||||
c.maxMonID++
|
||||
mons = append(mons, c.newMonConfig(c.maxMonID))
|
||||
}
|
||||
|
||||
return mons
|
||||
return existingCount, mons
|
||||
}
|
||||
|
||||
func (c *Cluster) newMonConfig(monID int) *monConfig {
|
||||
@@ -338,7 +367,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) startDeployments(mons []*monConfig, index int) error {
|
||||
func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) error {
|
||||
if len(mons) == 0 {
|
||||
return fmt.Errorf("cannot start 0 mons")
|
||||
}
|
||||
@@ -353,10 +382,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, index int) error {
|
||||
}
|
||||
|
||||
logger.Infof("mons created: %d", len(mons))
|
||||
return c.waitForMonsToJoin(mons)
|
||||
return c.waitForMonsToJoin(mons, requireAllInQuorum)
|
||||
}
|
||||
|
||||
func (c *Cluster) waitForMonsToJoin(mons []*monConfig) error {
|
||||
func (c *Cluster) waitForMonsToJoin(mons []*monConfig, requireAllInQuorum bool) error {
|
||||
if !c.waitForStart {
|
||||
return nil
|
||||
}
|
||||
@@ -367,7 +396,8 @@ func (c *Cluster) waitForMonsToJoin(mons []*monConfig) error {
|
||||
}
|
||||
|
||||
// wait for the monitors to join quorum
|
||||
err := waitForQuorumWithMons(c.context, c.clusterInfo.Name, starting)
|
||||
sleepTime := 5
|
||||
err := waitForQuorumWithMons(c.context, c.clusterInfo.Name, starting, sleepTime, requireAllInQuorum)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to wait for mon quorum. %+v", err)
|
||||
}
|
||||
@@ -454,13 +484,12 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons []string) error {
|
||||
func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons []string, sleepTime int, requireAllInQuorum bool) error {
|
||||
logger.Infof("waiting for mon quorum with %v", mons)
|
||||
|
||||
// wait for monitors to establish quorum
|
||||
retryCount := 0
|
||||
retryMax := 20
|
||||
sleepTime := 5
|
||||
for {
|
||||
retryCount++
|
||||
if retryCount > retryMax {
|
||||
@@ -473,13 +502,24 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [
|
||||
}
|
||||
|
||||
// wait for the mon pods to be running
|
||||
running, err := k8sutil.PodsRunningWithLabel(context.Clientset, clusterName, "app="+appName)
|
||||
if err != nil {
|
||||
logger.Infof("failed to query mon pod status, trying again. %+v", err)
|
||||
continue
|
||||
allPodsRunning := true
|
||||
var runningMonNames []string
|
||||
for _, m := range mons {
|
||||
running, err := k8sutil.PodsRunningWithLabel(context.Clientset, clusterName, fmt.Sprintf("app=%s,mon=%s", appName, m))
|
||||
if err != nil {
|
||||
logger.Infof("failed to query mon pod status, trying again. %+v", err)
|
||||
continue
|
||||
}
|
||||
if running > 0 {
|
||||
runningMonNames = append(runningMonNames, m)
|
||||
} else {
|
||||
allPodsRunning = false
|
||||
logger.Infof("mon %s is not yet running", m)
|
||||
}
|
||||
}
|
||||
if running != len(mons) {
|
||||
logger.Infof("%d/%d mon pods are running. waiting for pods to start", running, len(mons))
|
||||
|
||||
logger.Infof("mons running: %v", runningMonNames)
|
||||
if !allPodsRunning && requireAllInQuorum {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -491,49 +531,64 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [
|
||||
continue
|
||||
}
|
||||
|
||||
if !requireAllInQuorum {
|
||||
logQuorumMembers(monStatusResp)
|
||||
break
|
||||
}
|
||||
|
||||
// check if each of the initial monitors is in quorum
|
||||
allInQuorum := true
|
||||
for _, name := range mons {
|
||||
// first get the initial monitors corresponding mon map entry
|
||||
var monMapEntry *client.MonMapEntry
|
||||
for i := range monStatusResp.MonMap.Mons {
|
||||
if name == monStatusResp.MonMap.Mons[i].Name {
|
||||
monMapEntry = &monStatusResp.MonMap.Mons[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if monMapEntry == nil {
|
||||
// found an initial monitor that is not in the mon map, bail out of this retry
|
||||
logger.Warningf("failed to find initial monitor %s in mon map", name)
|
||||
allInQuorum = false
|
||||
break
|
||||
}
|
||||
|
||||
// using the current initial monitor's mon map entry, check to see if it's in the quorum list
|
||||
// (a list of monitor rank values)
|
||||
inQuorumList := false
|
||||
for _, q := range monStatusResp.Quorum {
|
||||
if monMapEntry.Rank == q {
|
||||
inQuorumList = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !inQuorumList {
|
||||
if !monFoundInQuorum(name, monStatusResp) {
|
||||
// found an initial monitor that is not in quorum, bail out of this retry
|
||||
logger.Warningf("initial monitor %s is not in quorum list", name)
|
||||
logger.Warningf("monitor %s is not in quorum list", name)
|
||||
allInQuorum = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if allInQuorum {
|
||||
logger.Debugf("all initial monitors are in quorum")
|
||||
logQuorumMembers(monStatusResp)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("Ceph monitors formed quorum")
|
||||
return nil
|
||||
}
|
||||
|
||||
func logQuorumMembers(monStatusResp client.MonStatusResponse) {
|
||||
var monsInQuorum []string
|
||||
for _, m := range monStatusResp.MonMap.Mons {
|
||||
if monFoundInQuorum(m.Name, monStatusResp) {
|
||||
monsInQuorum = append(monsInQuorum, m.Name)
|
||||
}
|
||||
}
|
||||
logger.Infof("Monitors in quorum: %v", monsInQuorum)
|
||||
}
|
||||
|
||||
func monFoundInQuorum(name string, monStatusResp client.MonStatusResponse) bool {
|
||||
// first get the initial monitors corresponding mon map entry
|
||||
var monMapEntry *client.MonMapEntry
|
||||
for i := range monStatusResp.MonMap.Mons {
|
||||
if name == monStatusResp.MonMap.Mons[i].Name {
|
||||
monMapEntry = &monStatusResp.MonMap.Mons[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if monMapEntry == nil {
|
||||
// found an initial monitor that is not in the mon map, bail out of this retry
|
||||
logger.Warningf("failed to find initial monitor %s in mon map", name)
|
||||
return false
|
||||
}
|
||||
|
||||
// using the current initial monitor's mon map entry, check to see if it's in the quorum list
|
||||
// (a list of monitor rank values)
|
||||
for _, q := range monStatusResp.Quorum {
|
||||
if monMapEntry.Rank == q {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
|
||||
@@ -70,6 +71,13 @@ func testGenMonConfig(monID string) *monConfig {
|
||||
}
|
||||
|
||||
func newTestStartCluster(namespace string) *clusterd.Context {
|
||||
monResponse := func() (string, error) {
|
||||
return clienttest.MonInQuorumResponseMany(3), nil
|
||||
}
|
||||
return newTestStartClusterWithQuorumResponse(namespace, monResponse)
|
||||
}
|
||||
|
||||
func newTestStartClusterWithQuorumResponse(namespace string, monResponse func() (string, error)) *clusterd.Context {
|
||||
clientset := test.New(3)
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
@@ -82,7 +90,7 @@ func newTestStartCluster(namespace string) *clusterd.Context {
|
||||
},
|
||||
MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
|
||||
// mock quorum health check because a second `Start()` triggers a health check
|
||||
return clienttest.MonInQuorumResponseMany(3), nil
|
||||
return monResponse()
|
||||
},
|
||||
}
|
||||
return &clusterd.Context{
|
||||
@@ -269,3 +277,47 @@ func TestNameToIndex(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 123, id)
|
||||
}
|
||||
|
||||
func TestWaitForQuorum(t *testing.T) {
|
||||
namespace := "ns"
|
||||
quorumChecks := 0
|
||||
quorumResponse := func() (string, error) {
|
||||
mons := map[string]*cephconfig.MonInfo{
|
||||
"a": {},
|
||||
}
|
||||
quorumChecks++
|
||||
if quorumChecks == 1 {
|
||||
// return an error the first time while we're waiting for the mon to join quorum
|
||||
return "", fmt.Errorf("test error")
|
||||
}
|
||||
// a successful response indicates that we have quorum, even if we didn't check which specific mons were in quorum
|
||||
return clienttest.MonInQuorumResponseFromMons(mons), nil
|
||||
}
|
||||
context := newTestStartClusterWithQuorumResponse(namespace, quorumResponse)
|
||||
requireAllInQuorum := false
|
||||
expectedMons := []string{"a"}
|
||||
err := waitForQuorumWithMons(context, namespace, expectedMons, 0, requireAllInQuorum)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestMonFoundInQuorum(t *testing.T) {
|
||||
response := client.MonStatusResponse{}
|
||||
|
||||
// "a" is in quorum
|
||||
response.Quorum = []int{0}
|
||||
response.MonMap.Mons = []client.MonMapEntry{
|
||||
{Name: "a", Rank: 0},
|
||||
{Name: "b", Rank: 1},
|
||||
{Name: "c", Rank: 2},
|
||||
}
|
||||
assert.True(t, monFoundInQuorum("a", response))
|
||||
assert.False(t, monFoundInQuorum("b", response))
|
||||
assert.False(t, monFoundInQuorum("c", response))
|
||||
|
||||
// b and c also in quorum, but not d
|
||||
response.Quorum = []int{0, 1, 2}
|
||||
assert.True(t, monFoundInQuorum("a", response))
|
||||
assert.True(t, monFoundInQuorum("b", response))
|
||||
assert.True(t, monFoundInQuorum("c", response))
|
||||
assert.False(t, monFoundInQuorum("d", response))
|
||||
}
|
||||
@@ -66,6 +66,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
logger.Infof("mon %s running at %s:%d", mon.DaemonName, s.Spec.ClusterIP, mon.Port)
|
||||
logger.Infof("mon %s endpoint is %s:%d", mon.DaemonName, s.Spec.ClusterIP, mon.Port)
|
||||
return s.Spec.ClusterIP, nil
|
||||
}
|
||||
Reference in new issue
Block a user