mon: improve mon failover reliability
Mon failover reliability is improved in two scenarios with these changes. First, if the failover is aborted because of failing to start the new mon and wait for it to join quorum, the new mon will be fully removed and the failover rolled back. Previously, the operator would leave the new mon pod running even if it never joined quorum. Now the timeout for the failover is increased to wait longer for the new mon, but if it times out, there are no leftovers from the cancelled failover. Second, if the operator is failed in the middle of a failover before the old mon is removed, the operator needs to remove an extra mon. The operator was only selecting a random mon for removal instead of honoring the stretch cluster topology or looking for mons on the same node that could be removed to restore the correct desired number of mons. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
3 files changed
+198
-29
No files matched your search
@@ -320,7 +320,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
logger.Warningf("cannot reduce mon quorum size from 2 to 1")
|
||||
} else {
|
||||
logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount)
|
||||
return c.removeMon(quorumStatus.MonMap.Mons[0].Name)
|
||||
return c.removeMon(c.determineExtraMonToRemove())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -341,6 +341,80 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// determineExtraMonToRemove assumes all mons are in quorum and that there are more mons
|
||||
// that required for desired state. One mon will be picked for removal in this priority:
|
||||
// 1. If a stretch cluster, remove the extra mon according to the stretch topology
|
||||
// 2. If more than one mon on a node, remove one of them
|
||||
// 3. If no criteria require for 1 or 2, pick an arbitrary mon
|
||||
func (c *Cluster) determineExtraMonToRemove() string {
|
||||
mons := c.clusterInfoToMonConfig()
|
||||
if c.spec.IsStretchCluster() {
|
||||
stretchMonToRemove := c.findExtraMonToRemoveFromStretchCluster(mons)
|
||||
if stretchMonToRemove != "" {
|
||||
return stretchMonToRemove
|
||||
}
|
||||
logger.Infof("did not find an extra mon to remove from the stretch cluster")
|
||||
return ""
|
||||
}
|
||||
|
||||
nodesWithMons := map[string]string{}
|
||||
arbitraryMon := ""
|
||||
for _, mon := range mons {
|
||||
if mon.NodeName == "" {
|
||||
logger.Debugf("mon %q is not scheduled to a specific host", mon.DaemonName)
|
||||
continue
|
||||
}
|
||||
// Check if there are multiple mons on the node
|
||||
if existingMon, ok := nodesWithMons[mon.NodeName]; ok {
|
||||
logger.Infof("found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName)
|
||||
return mon.DaemonName
|
||||
}
|
||||
nodesWithMons[mon.NodeName] = mon.DaemonName
|
||||
|
||||
// assign the current mon as the fallback mon
|
||||
arbitraryMon = mon.DaemonName
|
||||
}
|
||||
|
||||
logger.Infof("removing arbitrary extra mon %q", arbitraryMon)
|
||||
return arbitraryMon
|
||||
}
|
||||
|
||||
func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) string {
|
||||
// Build the count of current mons per zone
|
||||
zoneCount := map[string]int{}
|
||||
monInZones := map[string]string{}
|
||||
for _, m := range mons {
|
||||
if m.Zone == "" {
|
||||
logger.Warningf("zone not found on mon %q", m.DaemonName)
|
||||
continue
|
||||
}
|
||||
zoneCount[m.Zone]++
|
||||
// We just need the name of one of the mons in the zone in case there are extra
|
||||
monInZones[m.Zone] = m.DaemonName
|
||||
}
|
||||
|
||||
// Find a zone that has too many mons
|
||||
for _, zone := range c.spec.Mon.StretchCluster.Zones {
|
||||
count, ok := zoneCount[zone.Name]
|
||||
if !ok {
|
||||
// The zone isn't currently assigned to any mon, so skip it
|
||||
continue
|
||||
}
|
||||
if zone.Arbiter {
|
||||
if count > 1 {
|
||||
logger.Infof("removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name)
|
||||
return monInZones[zone.Name]
|
||||
}
|
||||
} else {
|
||||
if count > 2 {
|
||||
logger.Infof("removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name)
|
||||
return monInZones[zone.Name]
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// failMon compares the monCount against desiredMonCount
|
||||
// Returns whether the failover request was attempted. If false,
|
||||
// the operator should check for other mons to failover.
|
||||
@@ -459,25 +533,8 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
|
||||
func (c *Cluster) failoverMon(name string) error {
|
||||
logger.Infof("Failing over monitor %q", name)
|
||||
|
||||
// Scale down the failed mon to allow a new one to start
|
||||
if err := c.updateMonDeploymentReplica(name, false); err != nil {
|
||||
// attempt to continue with the failover even if the bad mon could not be stopped
|
||||
logger.Warningf("failed to stop mon %q for failover. %v", name, err)
|
||||
}
|
||||
newMonSucceeded := false
|
||||
defer func() {
|
||||
if newMonSucceeded {
|
||||
// do nothing if the new mon was started successfully, the deployment will anyway be deleted
|
||||
return
|
||||
}
|
||||
if err := c.updateMonDeploymentReplica(name, true); err != nil {
|
||||
// attempt to continue even if the bad mon could not be restarted
|
||||
logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
|
||||
}
|
||||
}()
|
||||
|
||||
// remove the failed mon from a local list of the existing mons for finding a stretch zone
|
||||
existingMons := c.clusterInfoToMonConfig(name)
|
||||
existingMons := c.clusterInfoToMonConfigWithExclude(name)
|
||||
|
||||
zone, err := c.findAvailableZoneIfStretched(existingMons)
|
||||
if err != nil {
|
||||
@@ -488,9 +545,40 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
m := c.newMonConfig(c.maxMonID+1, zone)
|
||||
logger.Infof("starting new mon: %+v", m)
|
||||
|
||||
mConf := []*monConfig{m}
|
||||
// Scale down the failed mon to allow a new one to start
|
||||
if err := c.updateMonDeploymentReplica(name, false); err != nil {
|
||||
// attempt to continue with the failover even if the bad mon could not be stopped
|
||||
logger.Warningf("failed to stop mon %q for failover. %v", name, err)
|
||||
}
|
||||
|
||||
// If the mon failover is not successful, revert the failover
|
||||
newMonSucceeded := false
|
||||
newMonMightBeInQuorum := false
|
||||
defer func() {
|
||||
if newMonSucceeded {
|
||||
// do nothing if the new mon was started successfully, the deployment will anyway be deleted
|
||||
return
|
||||
}
|
||||
logger.Warningf("failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName)
|
||||
if err := c.updateMonDeploymentReplica(name, true); err != nil {
|
||||
// attempt to continue even if the bad mon could not be restarted
|
||||
logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
|
||||
}
|
||||
if err := c.removeMonWithOptionalQuorum(m.DaemonName, newMonMightBeInQuorum); err != nil {
|
||||
logger.Infof("failed to remove mon %q from quorum. %v", m.DaemonName, err)
|
||||
}
|
||||
|
||||
// Make sure the maxMonID is reverted to its previous value
|
||||
// The maxMonId is committed to a configmap immediately after the mon deployment
|
||||
// is started, even though c.maxMonID is not incremented until the mon failover is successful
|
||||
logger.Infof("reverting maxMonId to %d", c.maxMonID)
|
||||
if err := c.commitMaxMonIDRequireIncrementing(c.maxMonID, false); err != nil {
|
||||
logger.Errorf("failed to revert maxMonId after starting mon %q", m.DaemonName)
|
||||
}
|
||||
}()
|
||||
|
||||
// Assign the pod to a node
|
||||
mConf := []*monConfig{m}
|
||||
if err := c.assignMons(mConf); err != nil {
|
||||
return errors.Wrap(err, "failed to place new mon on a node")
|
||||
}
|
||||
@@ -512,6 +600,7 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
c.ClusterInfo.Monitors[m.DaemonName] = cephclient.NewMonInfo(m.DaemonName, m.PublicIP, m.Port)
|
||||
|
||||
// Start the deployment
|
||||
newMonMightBeInQuorum = true
|
||||
if err := c.startDeployments(mConf, true); err != nil {
|
||||
return errors.Wrapf(err, "failed to start new mon %s", m.DaemonName)
|
||||
}
|
||||
@@ -533,6 +622,15 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
|
||||
// make a best effort to remove the mon and all its resources
|
||||
func (c *Cluster) removeMon(daemonName string) error {
|
||||
return c.removeMonWithOptionalQuorum(daemonName, true)
|
||||
}
|
||||
|
||||
// make a best effort to remove the mon and all its resources
|
||||
func (c *Cluster) removeMonWithOptionalQuorum(daemonName string, shouldRemoveFromQuorum bool) error {
|
||||
if daemonName == "" {
|
||||
logger.Info("did not identify a mon to remove")
|
||||
return nil
|
||||
}
|
||||
logger.Infof("ensuring removal of unhealthy monitor %s", daemonName)
|
||||
|
||||
resourceName := resourceName(daemonName)
|
||||
@@ -550,8 +648,10 @@ func (c *Cluster) removeMon(daemonName string) error {
|
||||
}
|
||||
|
||||
// Remove the bad monitor from quorum
|
||||
if err := c.removeMonitorFromQuorum(daemonName); err != nil {
|
||||
logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err)
|
||||
if shouldRemoveFromQuorum {
|
||||
if err := c.removeMonitorFromQuorum(daemonName); err != nil {
|
||||
logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err)
|
||||
}
|
||||
}
|
||||
delete(c.ClusterInfo.Monitors, daemonName)
|
||||
|
||||
|
||||
@@ -150,6 +150,62 @@ func TestCheckHealth(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveExtraMon(t *testing.T) {
|
||||
endpoint := "1.2.3.4:6789"
|
||||
c := &Cluster{mapping: &opcontroller.Mapping{}}
|
||||
c.ClusterInfo = &cephclient.ClusterInfo{Monitors: map[string]*cephclient.MonInfo{
|
||||
"a": {Name: "a", Endpoint: endpoint},
|
||||
"b": {Name: "b", Endpoint: endpoint},
|
||||
"c": {Name: "c", Endpoint: endpoint},
|
||||
"d": {Name: "d", Endpoint: endpoint},
|
||||
}}
|
||||
c.mapping.Schedule = map[string]*opcontroller.MonScheduleInfo{
|
||||
"a": {Name: "node1"},
|
||||
"b": {Name: "node2"},
|
||||
"c": {Name: "node3"},
|
||||
"d": {Name: "node1"},
|
||||
}
|
||||
// Remove mon an extra mon on the same node
|
||||
removedMon := c.determineExtraMonToRemove()
|
||||
if removedMon != "a" && removedMon != "d" {
|
||||
assert.Fail(t, fmt.Sprintf("removed mon %q instead of a or d", removedMon))
|
||||
}
|
||||
|
||||
// Remove an arbitrary mon that are all on different nodes
|
||||
c.mapping.Schedule["d"].Name = "node4"
|
||||
removedMon = c.determineExtraMonToRemove()
|
||||
assert.NotEqual(t, "", removedMon)
|
||||
|
||||
// Don't remove any extra mon from a proper stretch cluster
|
||||
c.spec.Mon.StretchCluster = &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
|
||||
{Name: "x", Arbiter: true},
|
||||
{Name: "y"},
|
||||
{Name: "z"},
|
||||
}}
|
||||
c.ClusterInfo.Monitors["e"] = &cephclient.MonInfo{Name: "e", Endpoint: endpoint}
|
||||
c.mapping.Schedule["a"].Zone = "x"
|
||||
c.mapping.Schedule["b"].Zone = "y"
|
||||
c.mapping.Schedule["c"].Zone = "y"
|
||||
c.mapping.Schedule["d"].Zone = "z"
|
||||
c.mapping.Schedule["e"] = &opcontroller.MonScheduleInfo{Name: "node5", Zone: "z"}
|
||||
removedMon = c.determineExtraMonToRemove()
|
||||
assert.Equal(t, "", removedMon)
|
||||
|
||||
// Remove an extra mon from the arbiter zone
|
||||
c.mapping.Schedule["d"].Zone = "x"
|
||||
removedMon = c.determineExtraMonToRemove()
|
||||
if removedMon != "a" && removedMon != "d" {
|
||||
assert.Fail(t, "removed mon %q instead of a or d from the arbiter zone", removedMon)
|
||||
}
|
||||
|
||||
// Remove an extra mon from a non-arbiter zone
|
||||
c.mapping.Schedule["d"].Zone = "y"
|
||||
removedMon = c.determineExtraMonToRemove()
|
||||
if removedMon != "b" && removedMon != "c" && removedMon != "d" {
|
||||
assert.Fail(t, fmt.Sprintf("removed mon %q instead of b, c, or d from the non-arbiter zone", removedMon))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkipMonFailover(t *testing.T) {
|
||||
c := New(context.TODO(), &clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil)
|
||||
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
||||
|
||||
@@ -127,6 +127,8 @@ type monConfig struct {
|
||||
Port int32
|
||||
// The zone used for a stretch cluster
|
||||
Zone string
|
||||
// The node where the mon is assigned
|
||||
NodeName string
|
||||
// DataPathMap is the mapping relationship between mon data stored on the host and mon data
|
||||
// stored in containers.
|
||||
DataPathMap *config.DataPathMap
|
||||
@@ -515,7 +517,7 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s
|
||||
func (c *Cluster) initMonConfig(size int) (int, []*monConfig, error) {
|
||||
|
||||
// initialize the mon pod info for mons that have been previously created
|
||||
mons := c.clusterInfoToMonConfig("")
|
||||
mons := c.clusterInfoToMonConfig()
|
||||
|
||||
// initialize mon info if we don't have enough mons (at first startup)
|
||||
existingCount := len(c.ClusterInfo.Monitors)
|
||||
@@ -531,7 +533,11 @@ func (c *Cluster) initMonConfig(size int) (int, []*monConfig, error) {
|
||||
return existingCount, mons, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig {
|
||||
func (c *Cluster) clusterInfoToMonConfig() []*monConfig {
|
||||
return c.clusterInfoToMonConfigWithExclude("")
|
||||
}
|
||||
|
||||
func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monConfig {
|
||||
mons := []*monConfig{}
|
||||
for _, monitor := range c.ClusterInfo.Monitors {
|
||||
if monitor.Name == excludedMon {
|
||||
@@ -539,9 +545,11 @@ func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig {
|
||||
continue
|
||||
}
|
||||
var zone string
|
||||
var nodeName string
|
||||
schedule := c.mapping.Schedule[monitor.Name]
|
||||
if schedule != nil {
|
||||
zone = schedule.Zone
|
||||
nodeName = schedule.Name
|
||||
}
|
||||
mons = append(mons, &monConfig{
|
||||
ResourceName: resourceName(monitor.Name),
|
||||
@@ -549,6 +557,7 @@ func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig {
|
||||
Port: cephutil.GetPortFromEndpoint(monitor.Endpoint),
|
||||
PublicIP: cephutil.GetIPFromEndpoint(monitor.Endpoint),
|
||||
Zone: zone,
|
||||
NodeName: nodeName,
|
||||
DataPathMap: config.NewStatefulDaemonDataPathMap(c.spec.DataDirHostPath, dataDirRelativeHostPath(monitor.Name), config.MonType, monitor.Name, c.Namespace),
|
||||
})
|
||||
}
|
||||
@@ -1125,6 +1134,10 @@ func (c *Cluster) commitMaxMonID(monName string) error {
|
||||
return errors.Wrapf(err, "invalid mon name %q", monName)
|
||||
}
|
||||
|
||||
return c.commitMaxMonIDRequireIncrementing(committedMonID, true)
|
||||
}
|
||||
|
||||
func (c *Cluster) commitMaxMonIDRequireIncrementing(desiredMaxMonID int, requireIncrementing bool) error {
|
||||
configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(c.ClusterInfo.Context, EndpointConfigMapName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to find existing mon endpoint config map")
|
||||
@@ -1136,13 +1149,13 @@ func (c *Cluster) commitMaxMonID(monName string) error {
|
||||
return errors.Wrap(err, "failed to read existing maxMonId")
|
||||
}
|
||||
|
||||
if existingMax >= committedMonID {
|
||||
logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", committedMonID, existingMax)
|
||||
if requireIncrementing && existingMax >= desiredMaxMonID {
|
||||
logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("updating maxMonID from %d to %d after committing mon %q", existingMax, committedMonID, monName)
|
||||
configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(committedMonID)
|
||||
logger.Infof("updating maxMonID from %d to %d", existingMax, desiredMaxMonID)
|
||||
configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(desiredMaxMonID)
|
||||
|
||||
if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil {
|
||||
return errors.Wrap(err, "failed to update mon endpoint config map for the maxMonID")
|
||||
@@ -1327,7 +1340,7 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
|
||||
|
||||
// wait for monitors to establish quorum
|
||||
retryCount := 0
|
||||
retryMax := 30
|
||||
retryMax := 60
|
||||
for {
|
||||
// Return immediately if the context has been canceled
|
||||
if clusterInfo.Context.Err() != nil {
|
||||
|
||||
Reference in new issue
Block a user