handle ceph mon failover more gracefully

Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
travisn committed 2019-02-11 17:35:28 -07:00
1 parent e1dccd86e4
commit b72c6436ef
5 files changed
+184 -77

No files matched your search

@@ -462,7 +462,7 @@ spec:
# The duration to wait before trying to failover or remove/replace the
# current mon with a new mon (useful for compensating flapping network).
- name: ROOK_MON_OUT_TIMEOUT
value: "300s"
value: "600s"
# The duration between discovering devices in the rook-discover daemonset.
- name: ROOK_DISCOVER_DEVICES_INTERVAL
value: "60m"
+2 -2
View File
@@ -134,7 +134,7 @@ func (c *Cluster) checkHealth() error {
// when the timeout for the mon has been reached, continue to the
// normal failover/delete mon pod part of the code
if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout {
logger.Warningf("mon %s not found in quorum, still in mon out timeout", mon.Name)
logger.Warningf("mon %s not found in quorum, waiting for timeout before failover", mon.Name)
continue
}
@@ -281,7 +281,7 @@ func (c *Cluster) failoverMon(name string) error {
c.clusterInfo.Monitors[m.DaemonName] = cephconfig.NewMonInfo(m.DaemonName, m.PublicIP, m.Port)
// Start the deployment
if err = c.startDeployments(mConf, len(mConf)-1); err != nil {
if err = c.startDeployments(mConf, true); err != nil {
return fmt.Errorf("failed to start new mon %s. %+v", m.DaemonName, err)
}
+127 -72
View File
@@ -176,48 +176,76 @@ func (c *Cluster) Start() error {
}
func (c *Cluster) startMons() error {
// init the mons config
mons := c.initMonConfig(c.Count)
// init the mon config
existingCount, mons := c.initMonConfig(c.Count)
// Assign the pods to nodes
// Assign the mons to nodes
if err := c.assignMons(mons); err != nil {
return fmt.Errorf("failed to assign pods to mons. %+v", err)
}
// Start one monitor at a time
for i := 0; i < c.Count; i++ {
logger.Infof("ensuring mon %s (%s) is started", mons[i].ResourceName, mons[i].DaemonName)
endIndex := len(c.clusterInfo.Monitors)
if endIndex < c.Count {
endIndex++
if existingCount < len(mons) {
// Start the new mons one at a time
for i := existingCount; i < c.Count; i++ {
if err := c.ensureMonsRunning(mons, i, true); err != nil {
return err
}
}
logger.Infof("looping to start mons. i=%d, endIndex=%d, c.Size=%d", i, endIndex, c.Count)
// Init the mon IPs
if err := c.initMonIPs(mons[0:endIndex]); err != nil {
return fmt.Errorf("failed to init mon services. %+v", err)
}
// save the mon config after we have "initiated the IPs"
if err := c.saveMonConfig(); err != nil {
return fmt.Errorf("failed to save mons. %+v", err)
}
// make sure we have the connection info generated so connections can happen
if err := writeConnectionConfig(c.context, c.clusterInfo); err != nil {
} else {
// Ensure all the expected mon deployments exist, but don't require full quorum to continue
lastMonIndex := len(mons) - 1
if err := c.ensureMonsRunning(mons, lastMonIndex, false); err != nil {
return err
}
// Start the deployment
if err := c.startDeployments(mons[0:endIndex], i); err != nil {
return fmt.Errorf("failed to start mon pods. %+v", err)
}
}
logger.Debugf("mon endpoints used are: %s", FlattenMonEndpoints(c.clusterInfo.Monitors))
return nil
}
// ensureMonsRunning is called in two scenarios:
// 1. To create a new mon and wait for it to join quorum (requireAllInQuorum = true). This method will be called multiple times
// to add a mon until we have reached the desired number of mons.
// 2. To check that the majority of existing mons are in quorum. It is ok if not all mons are in quorum. (requireAllInQuorum = false)
// This is needed when the operator is restarted and all mons may not be up or in quorum.
func (c *Cluster) ensureMonsRunning(mons []*monConfig, i int, requireAllInQuorum bool) error {
if requireAllInQuorum {
logger.Infof("creating mon %s", mons[i].DaemonName)
} else {
logger.Info("checking for basic quorum with existing mons")
}
// Calculate how many mons we expected to exist after this method is completed.
// If we are adding a new mon, we expect one more than currently exist.
// If we haven't created all the desired mons already, we will be adding a new one with this iteration
expectedMonCount := len(c.clusterInfo.Monitors)
if expectedMonCount < c.Count {
expectedMonCount++
}
// Init the mon IPs
if err := c.initMonIPs(mons[0:expectedMonCount]); err != nil {
return fmt.Errorf("failed to init mon services. %+v", err)
}
// save the mon config after we have "initiated the IPs"
if err := c.saveMonConfig(); err != nil {
return fmt.Errorf("failed to save mons. %+v", err)
}
// make sure we have the connection info generated so connections can happen
if err := writeConnectionConfig(c.context, c.clusterInfo); err != nil {
return err
}
// Start the deployment
if err := c.startDeployments(mons[0:expectedMonCount], requireAllInQuorum); err != nil {
return fmt.Errorf("failed to start mon pods. %+v", err)
}
return nil
}
// initClusterInfo retrieves the ceph cluster info if it already exists.
// If a new cluster, create new keys.
func (c *Cluster) initClusterInfo() error {
@@ -240,7 +268,7 @@ func (c *Cluster) initClusterInfo() error {
return nil
}
func (c *Cluster) initMonConfig(size int) []*monConfig {
func (c *Cluster) initMonConfig(size int) (int, []*monConfig) {
mons := []*monConfig{}
// initialize the mon pod info for mons that have been previously created
@@ -255,12 +283,13 @@ func (c *Cluster) initMonConfig(size int) []*monConfig {
}
// initialize mon info if we don't have enough mons (at first startup)
existingCount := len(c.clusterInfo.Monitors)
for i := len(c.clusterInfo.Monitors); i < size; i++ {
c.maxMonID++
mons = append(mons, c.newMonConfig(c.maxMonID))
}
return mons
return existingCount, mons
}
func (c *Cluster) newMonConfig(monID int) *monConfig {
@@ -338,7 +367,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
return nil
}
func (c *Cluster) startDeployments(mons []*monConfig, index int) error {
func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) error {
if len(mons) == 0 {
return fmt.Errorf("cannot start 0 mons")
}
@@ -353,10 +382,10 @@ func (c *Cluster) startDeployments(mons []*monConfig, index int) error {
}
logger.Infof("mons created: %d", len(mons))
return c.waitForMonsToJoin(mons)
return c.waitForMonsToJoin(mons, requireAllInQuorum)
}
func (c *Cluster) waitForMonsToJoin(mons []*monConfig) error {
func (c *Cluster) waitForMonsToJoin(mons []*monConfig, requireAllInQuorum bool) error {
if !c.waitForStart {
return nil
}
@@ -367,7 +396,8 @@ func (c *Cluster) waitForMonsToJoin(mons []*monConfig) error {
}
// wait for the monitors to join quorum
err := waitForQuorumWithMons(c.context, c.clusterInfo.Name, starting)
sleepTime := 5
err := waitForQuorumWithMons(c.context, c.clusterInfo.Name, starting, sleepTime, requireAllInQuorum)
if err != nil {
return fmt.Errorf("failed to wait for mon quorum. %+v", err)
}
@@ -454,13 +484,12 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error {
return nil
}
func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons []string) error {
func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons []string, sleepTime int, requireAllInQuorum bool) error {
logger.Infof("waiting for mon quorum with %v", mons)
// wait for monitors to establish quorum
retryCount := 0
retryMax := 20
sleepTime := 5
for {
retryCount++
if retryCount > retryMax {
@@ -473,13 +502,24 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [
}
// wait for the mon pods to be running
running, err := k8sutil.PodsRunningWithLabel(context.Clientset, clusterName, "app="+appName)
if err != nil {
logger.Infof("failed to query mon pod status, trying again. %+v", err)
continue
allPodsRunning := true
var runningMonNames []string
for _, m := range mons {
running, err := k8sutil.PodsRunningWithLabel(context.Clientset, clusterName, fmt.Sprintf("app=%s,mon=%s", appName, m))
if err != nil {
logger.Infof("failed to query mon pod status, trying again. %+v", err)
continue
}
if running > 0 {
runningMonNames = append(runningMonNames, m)
} else {
allPodsRunning = false
logger.Infof("mon %s is not yet running", m)
}
}
if running != len(mons) {
logger.Infof("%d/%d mon pods are running. waiting for pods to start", running, len(mons))
logger.Infof("mons running: %v", runningMonNames)
if !allPodsRunning && requireAllInQuorum {
continue
}
@@ -491,49 +531,64 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [
continue
}
if !requireAllInQuorum {
logQuorumMembers(monStatusResp)
break
}
// check if each of the initial monitors is in quorum
allInQuorum := true
for _, name := range mons {
// first get the initial monitors corresponding mon map entry
var monMapEntry *client.MonMapEntry
for i := range monStatusResp.MonMap.Mons {
if name == monStatusResp.MonMap.Mons[i].Name {
monMapEntry = &monStatusResp.MonMap.Mons[i]
break
}
}
if monMapEntry == nil {
// found an initial monitor that is not in the mon map, bail out of this retry
logger.Warningf("failed to find initial monitor %s in mon map", name)
allInQuorum = false
break
}
// using the current initial monitor's mon map entry, check to see if it's in the quorum list
// (a list of monitor rank values)
inQuorumList := false
for _, q := range monStatusResp.Quorum {
if monMapEntry.Rank == q {
inQuorumList = true
break
}
}
if !inQuorumList {
if !monFoundInQuorum(name, monStatusResp) {
// found an initial monitor that is not in quorum, bail out of this retry
logger.Warningf("initial monitor %s is not in quorum list", name)
logger.Warningf("monitor %s is not in quorum list", name)
allInQuorum = false
break
}
}
if allInQuorum {
logger.Debugf("all initial monitors are in quorum")
logQuorumMembers(monStatusResp)
break
}
}
logger.Infof("Ceph monitors formed quorum")
return nil
}
func logQuorumMembers(monStatusResp client.MonStatusResponse) {
var monsInQuorum []string
for _, m := range monStatusResp.MonMap.Mons {
if monFoundInQuorum(m.Name, monStatusResp) {
monsInQuorum = append(monsInQuorum, m.Name)
}
}
logger.Infof("Monitors in quorum: %v", monsInQuorum)
}
func monFoundInQuorum(name string, monStatusResp client.MonStatusResponse) bool {
// first get the initial monitors corresponding mon map entry
var monMapEntry *client.MonMapEntry
for i := range monStatusResp.MonMap.Mons {
if name == monStatusResp.MonMap.Mons[i].Name {
monMapEntry = &monStatusResp.MonMap.Mons[i]
break
}
}
if monMapEntry == nil {
// found an initial monitor that is not in the mon map, bail out of this retry
logger.Warningf("failed to find initial monitor %s in mon map", name)
return false
}
// using the current initial monitor's mon map entry, check to see if it's in the quorum list
// (a list of monitor rank values)
for _, q := range monStatusResp.Quorum {
if monMapEntry.Rank == q {
return true
}
}
return false
}
+53 -1
View File
@@ -26,6 +26,7 @@ import (
"testing"
"time"
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -70,6 +71,13 @@ func testGenMonConfig(monID string) *monConfig {
}
func newTestStartCluster(namespace string) *clusterd.Context {
monResponse := func() (string, error) {
return clienttest.MonInQuorumResponseMany(3), nil
}
return newTestStartClusterWithQuorumResponse(namespace, monResponse)
}
func newTestStartClusterWithQuorumResponse(namespace string, monResponse func() (string, error)) *clusterd.Context {
clientset := test.New(3)
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
@@ -82,7 +90,7 @@ func newTestStartCluster(namespace string) *clusterd.Context {
},
MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
// mock quorum health check because a second `Start()` triggers a health check
return clienttest.MonInQuorumResponseMany(3), nil
return monResponse()
},
}
return &clusterd.Context{
@@ -269,3 +277,47 @@ func TestNameToIndex(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, 123, id)
}
func TestWaitForQuorum(t *testing.T) {
namespace := "ns"
quorumChecks := 0
quorumResponse := func() (string, error) {
mons := map[string]*cephconfig.MonInfo{
"a": {},
}
quorumChecks++
if quorumChecks == 1 {
// return an error the first time while we're waiting for the mon to join quorum
return "", fmt.Errorf("test error")
}
// a successful response indicates that we have quorum, even if we didn't check which specific mons were in quorum
return clienttest.MonInQuorumResponseFromMons(mons), nil
}
context := newTestStartClusterWithQuorumResponse(namespace, quorumResponse)
requireAllInQuorum := false
expectedMons := []string{"a"}
err := waitForQuorumWithMons(context, namespace, expectedMons, 0, requireAllInQuorum)
assert.Nil(t, err)
}
func TestMonFoundInQuorum(t *testing.T) {
response := client.MonStatusResponse{}
// "a" is in quorum
response.Quorum = []int{0}
response.MonMap.Mons = []client.MonMapEntry{
{Name: "a", Rank: 0},
{Name: "b", Rank: 1},
{Name: "c", Rank: 2},
}
assert.True(t, monFoundInQuorum("a", response))
assert.False(t, monFoundInQuorum("b", response))
assert.False(t, monFoundInQuorum("c", response))
// b and c also in quorum, but not d
response.Quorum = []int{0, 1, 2}
assert.True(t, monFoundInQuorum("a", response))
assert.True(t, monFoundInQuorum("b", response))
assert.True(t, monFoundInQuorum("c", response))
assert.False(t, monFoundInQuorum("d", response))
}
+1 -1
View File
@@ -66,6 +66,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) {
return "", nil
}
logger.Infof("mon %s running at %s:%d", mon.DaemonName, s.Spec.ClusterIP, mon.Port)
logger.Infof("mon %s endpoint is %s:%d", mon.DaemonName, s.Spec.ClusterIP, mon.Port)
return s.Spec.ClusterIP, nil
}