From b72c6436ef99640ca03a53643f97342dde81d56d Mon Sep 17 00:00:00 2001 From: travisn Date: Thu, 7 Feb 2019 15:19:33 -0700 Subject: [PATCH] handle ceph mon failover more gracefully Signed-off-by: travisn --- .../examples/kubernetes/ceph/operator.yaml | 2 +- pkg/operator/ceph/cluster/mon/health.go | 4 +- pkg/operator/ceph/cluster/mon/mon.go | 199 +++++++++++------- pkg/operator/ceph/cluster/mon/mon_test.go | 54 ++++- pkg/operator/ceph/cluster/mon/service.go | 2 +- 5 files changed, 184 insertions(+), 77 deletions(-) diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 25d2f8be3..4586a02ed 100755 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -462,7 +462,7 @@ spec: # The duration to wait before trying to failover or remove/replace the # current mon with a new mon (useful for compensating flapping network). - name: ROOK_MON_OUT_TIMEOUT - value: "300s" + value: "600s" # The duration between discovering devices in the rook-discover daemonset. - name: ROOK_DISCOVER_DEVICES_INTERVAL value: "60m" diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 63f2a29dd..f596393c9 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -134,7 +134,7 @@ func (c *Cluster) checkHealth() error { // when the timeout for the mon has been reached, continue to the // normal failover/delete mon pod part of the code if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout { - logger.Warningf("mon %s not found in quorum, still in mon out timeout", mon.Name) + logger.Warningf("mon %s not found in quorum, waiting for timeout before failover", mon.Name) continue } @@ -281,7 +281,7 @@ func (c *Cluster) failoverMon(name string) error { c.clusterInfo.Monitors[m.DaemonName] = cephconfig.NewMonInfo(m.DaemonName, m.PublicIP, m.Port) // Start the deployment - if err = c.startDeployments(mConf, len(mConf)-1); err != nil { + if err = c.startDeployments(mConf, true); err != nil { return fmt.Errorf("failed to start new mon %s. %+v", m.DaemonName, err) } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 64cda05b1..dd7fdd02c 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -176,48 +176,76 @@ func (c *Cluster) Start() error { } func (c *Cluster) startMons() error { - // init the mons config - mons := c.initMonConfig(c.Count) + // init the mon config + existingCount, mons := c.initMonConfig(c.Count) - // Assign the pods to nodes + // Assign the mons to nodes if err := c.assignMons(mons); err != nil { return fmt.Errorf("failed to assign pods to mons. %+v", err) } - // Start one monitor at a time - for i := 0; i < c.Count; i++ { - logger.Infof("ensuring mon %s (%s) is started", mons[i].ResourceName, mons[i].DaemonName) - endIndex := len(c.clusterInfo.Monitors) - if endIndex < c.Count { - endIndex++ + if existingCount < len(mons) { + // Start the new mons one at a time + for i := existingCount; i < c.Count; i++ { + if err := c.ensureMonsRunning(mons, i, true); err != nil { + return err + } } - logger.Infof("looping to start mons. i=%d, endIndex=%d, c.Size=%d", i, endIndex, c.Count) - - // Init the mon IPs - if err := c.initMonIPs(mons[0:endIndex]); err != nil { - return fmt.Errorf("failed to init mon services. %+v", err) - } - - // save the mon config after we have "initiated the IPs" - if err := c.saveMonConfig(); err != nil { - return fmt.Errorf("failed to save mons. %+v", err) - } - - // make sure we have the connection info generated so connections can happen - if err := writeConnectionConfig(c.context, c.clusterInfo); err != nil { + } else { + // Ensure all the expected mon deployments exist, but don't require full quorum to continue + lastMonIndex := len(mons) - 1 + if err := c.ensureMonsRunning(mons, lastMonIndex, false); err != nil { return err } - - // Start the deployment - if err := c.startDeployments(mons[0:endIndex], i); err != nil { - return fmt.Errorf("failed to start mon pods. %+v", err) - } } logger.Debugf("mon endpoints used are: %s", FlattenMonEndpoints(c.clusterInfo.Monitors)) return nil } +// ensureMonsRunning is called in two scenarios: +// 1. To create a new mon and wait for it to join quorum (requireAllInQuorum = true). This method will be called multiple times +// to add a mon until we have reached the desired number of mons. +// 2. To check that the majority of existing mons are in quorum. It is ok if not all mons are in quorum. (requireAllInQuorum = false) +// This is needed when the operator is restarted and all mons may not be up or in quorum. +func (c *Cluster) ensureMonsRunning(mons []*monConfig, i int, requireAllInQuorum bool) error { + if requireAllInQuorum { + logger.Infof("creating mon %s", mons[i].DaemonName) + } else { + logger.Info("checking for basic quorum with existing mons") + } + + // Calculate how many mons we expected to exist after this method is completed. + // If we are adding a new mon, we expect one more than currently exist. + // If we haven't created all the desired mons already, we will be adding a new one with this iteration + expectedMonCount := len(c.clusterInfo.Monitors) + if expectedMonCount < c.Count { + expectedMonCount++ + } + + // Init the mon IPs + if err := c.initMonIPs(mons[0:expectedMonCount]); err != nil { + return fmt.Errorf("failed to init mon services. %+v", err) + } + + // save the mon config after we have "initiated the IPs" + if err := c.saveMonConfig(); err != nil { + return fmt.Errorf("failed to save mons. %+v", err) + } + + // make sure we have the connection info generated so connections can happen + if err := writeConnectionConfig(c.context, c.clusterInfo); err != nil { + return err + } + + // Start the deployment + if err := c.startDeployments(mons[0:expectedMonCount], requireAllInQuorum); err != nil { + return fmt.Errorf("failed to start mon pods. %+v", err) + } + + return nil +} + // initClusterInfo retrieves the ceph cluster info if it already exists. // If a new cluster, create new keys. func (c *Cluster) initClusterInfo() error { @@ -240,7 +268,7 @@ func (c *Cluster) initClusterInfo() error { return nil } -func (c *Cluster) initMonConfig(size int) []*monConfig { +func (c *Cluster) initMonConfig(size int) (int, []*monConfig) { mons := []*monConfig{} // initialize the mon pod info for mons that have been previously created @@ -255,12 +283,13 @@ func (c *Cluster) initMonConfig(size int) []*monConfig { } // initialize mon info if we don't have enough mons (at first startup) + existingCount := len(c.clusterInfo.Monitors) for i := len(c.clusterInfo.Monitors); i < size; i++ { c.maxMonID++ mons = append(mons, c.newMonConfig(c.maxMonID)) } - return mons + return existingCount, mons } func (c *Cluster) newMonConfig(monID int) *monConfig { @@ -338,7 +367,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error { return nil } -func (c *Cluster) startDeployments(mons []*monConfig, index int) error { +func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) error { if len(mons) == 0 { return fmt.Errorf("cannot start 0 mons") } @@ -353,10 +382,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, index int) error { } logger.Infof("mons created: %d", len(mons)) - return c.waitForMonsToJoin(mons) + return c.waitForMonsToJoin(mons, requireAllInQuorum) } -func (c *Cluster) waitForMonsToJoin(mons []*monConfig) error { +func (c *Cluster) waitForMonsToJoin(mons []*monConfig, requireAllInQuorum bool) error { if !c.waitForStart { return nil } @@ -367,7 +396,8 @@ func (c *Cluster) waitForMonsToJoin(mons []*monConfig) error { } // wait for the monitors to join quorum - err := waitForQuorumWithMons(c.context, c.clusterInfo.Name, starting) + sleepTime := 5 + err := waitForQuorumWithMons(c.context, c.clusterInfo.Name, starting, sleepTime, requireAllInQuorum) if err != nil { return fmt.Errorf("failed to wait for mon quorum. %+v", err) } @@ -454,13 +484,12 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error { return nil } -func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons []string) error { +func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons []string, sleepTime int, requireAllInQuorum bool) error { logger.Infof("waiting for mon quorum with %v", mons) // wait for monitors to establish quorum retryCount := 0 retryMax := 20 - sleepTime := 5 for { retryCount++ if retryCount > retryMax { @@ -473,13 +502,24 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [ } // wait for the mon pods to be running - running, err := k8sutil.PodsRunningWithLabel(context.Clientset, clusterName, "app="+appName) - if err != nil { - logger.Infof("failed to query mon pod status, trying again. %+v", err) - continue + allPodsRunning := true + var runningMonNames []string + for _, m := range mons { + running, err := k8sutil.PodsRunningWithLabel(context.Clientset, clusterName, fmt.Sprintf("app=%s,mon=%s", appName, m)) + if err != nil { + logger.Infof("failed to query mon pod status, trying again. %+v", err) + continue + } + if running > 0 { + runningMonNames = append(runningMonNames, m) + } else { + allPodsRunning = false + logger.Infof("mon %s is not yet running", m) + } } - if running != len(mons) { - logger.Infof("%d/%d mon pods are running. waiting for pods to start", running, len(mons)) + + logger.Infof("mons running: %v", runningMonNames) + if !allPodsRunning && requireAllInQuorum { continue } @@ -491,49 +531,64 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [ continue } + if !requireAllInQuorum { + logQuorumMembers(monStatusResp) + break + } + // check if each of the initial monitors is in quorum allInQuorum := true for _, name := range mons { - // first get the initial monitors corresponding mon map entry - var monMapEntry *client.MonMapEntry - for i := range monStatusResp.MonMap.Mons { - if name == monStatusResp.MonMap.Mons[i].Name { - monMapEntry = &monStatusResp.MonMap.Mons[i] - break - } - } - - if monMapEntry == nil { - // found an initial monitor that is not in the mon map, bail out of this retry - logger.Warningf("failed to find initial monitor %s in mon map", name) - allInQuorum = false - break - } - - // using the current initial monitor's mon map entry, check to see if it's in the quorum list - // (a list of monitor rank values) - inQuorumList := false - for _, q := range monStatusResp.Quorum { - if monMapEntry.Rank == q { - inQuorumList = true - break - } - } - - if !inQuorumList { + if !monFoundInQuorum(name, monStatusResp) { // found an initial monitor that is not in quorum, bail out of this retry - logger.Warningf("initial monitor %s is not in quorum list", name) + logger.Warningf("monitor %s is not in quorum list", name) allInQuorum = false break } } if allInQuorum { - logger.Debugf("all initial monitors are in quorum") + logQuorumMembers(monStatusResp) break } } - logger.Infof("Ceph monitors formed quorum") return nil } + +func logQuorumMembers(monStatusResp client.MonStatusResponse) { + var monsInQuorum []string + for _, m := range monStatusResp.MonMap.Mons { + if monFoundInQuorum(m.Name, monStatusResp) { + monsInQuorum = append(monsInQuorum, m.Name) + } + } + logger.Infof("Monitors in quorum: %v", monsInQuorum) +} + +func monFoundInQuorum(name string, monStatusResp client.MonStatusResponse) bool { + // first get the initial monitors corresponding mon map entry + var monMapEntry *client.MonMapEntry + for i := range monStatusResp.MonMap.Mons { + if name == monStatusResp.MonMap.Mons[i].Name { + monMapEntry = &monStatusResp.MonMap.Mons[i] + break + } + } + + if monMapEntry == nil { + // found an initial monitor that is not in the mon map, bail out of this retry + logger.Warningf("failed to find initial monitor %s in mon map", name) + return false + } + + // using the current initial monitor's mon map entry, check to see if it's in the quorum list + // (a list of monitor rank values) + for _, q := range monStatusResp.Quorum { + if monMapEntry.Rank == q { + return true + } + } + + return false +} diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 03552af1d..0b21f7b6a 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" @@ -70,6 +71,13 @@ func testGenMonConfig(monID string) *monConfig { } func newTestStartCluster(namespace string) *clusterd.Context { + monResponse := func() (string, error) { + return clienttest.MonInQuorumResponseMany(3), nil + } + return newTestStartClusterWithQuorumResponse(namespace, monResponse) +} + +func newTestStartClusterWithQuorumResponse(namespace string, monResponse func() (string, error)) *clusterd.Context { clientset := test.New(3) configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) @@ -82,7 +90,7 @@ func newTestStartCluster(namespace string) *clusterd.Context { }, MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { // mock quorum health check because a second `Start()` triggers a health check - return clienttest.MonInQuorumResponseMany(3), nil + return monResponse() }, } return &clusterd.Context{ @@ -269,3 +277,47 @@ func TestNameToIndex(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 123, id) } + +func TestWaitForQuorum(t *testing.T) { + namespace := "ns" + quorumChecks := 0 + quorumResponse := func() (string, error) { + mons := map[string]*cephconfig.MonInfo{ + "a": {}, + } + quorumChecks++ + if quorumChecks == 1 { + // return an error the first time while we're waiting for the mon to join quorum + return "", fmt.Errorf("test error") + } + // a successful response indicates that we have quorum, even if we didn't check which specific mons were in quorum + return clienttest.MonInQuorumResponseFromMons(mons), nil + } + context := newTestStartClusterWithQuorumResponse(namespace, quorumResponse) + requireAllInQuorum := false + expectedMons := []string{"a"} + err := waitForQuorumWithMons(context, namespace, expectedMons, 0, requireAllInQuorum) + assert.Nil(t, err) +} + +func TestMonFoundInQuorum(t *testing.T) { + response := client.MonStatusResponse{} + + // "a" is in quorum + response.Quorum = []int{0} + response.MonMap.Mons = []client.MonMapEntry{ + {Name: "a", Rank: 0}, + {Name: "b", Rank: 1}, + {Name: "c", Rank: 2}, + } + assert.True(t, monFoundInQuorum("a", response)) + assert.False(t, monFoundInQuorum("b", response)) + assert.False(t, monFoundInQuorum("c", response)) + + // b and c also in quorum, but not d + response.Quorum = []int{0, 1, 2} + assert.True(t, monFoundInQuorum("a", response)) + assert.True(t, monFoundInQuorum("b", response)) + assert.True(t, monFoundInQuorum("c", response)) + assert.False(t, monFoundInQuorum("d", response)) +} diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index 2e11e0d68..d69de0344 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -66,6 +66,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { return "", nil } - logger.Infof("mon %s running at %s:%d", mon.DaemonName, s.Spec.ClusterIP, mon.Port) + logger.Infof("mon %s endpoint is %s:%d", mon.DaemonName, s.Spec.ClusterIP, mon.Port) return s.Spec.ClusterIP, nil }