mon: improve mon failover reliability

Mon failover reliability is improved in two scenarios with these changes.
First, if the failover is aborted because of failing to start the new mon
and wait for it to join quorum, the new mon will be fully removed and
the failover rolled back. Previously, the operator would leave the new
mon pod running even if it never joined quorum. Now the timeout for the failover
is increased to wait longer for the new mon, but if it times out, there
are no leftovers from the cancelled failover.

Second, if the operator is failed in the middle of a failover
before the old mon is removed, the operator needs to remove an extra
mon. The operator was only selecting a random mon for removal instead
of honoring the stretch cluster topology or looking for mons
on the same node that could be removed to restore the correct
desired number of mons.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2022-08-23 19:52:41 -06:00
1 parent cc15c94e6c
commit 72a4245fbf
3 files changed
+198 -29

No files matched your search

+122 -22
View File
@@ -320,7 +320,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
logger.Warningf("cannot reduce mon quorum size from 2 to 1")
} else {
logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount)
return c.removeMon(quorumStatus.MonMap.Mons[0].Name)
return c.removeMon(c.determineExtraMonToRemove())
}
}
@@ -341,6 +341,80 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
return nil
}
// determineExtraMonToRemove assumes all mons are in quorum and that there are more mons
// that required for desired state. One mon will be picked for removal in this priority:
// 1. If a stretch cluster, remove the extra mon according to the stretch topology
// 2. If more than one mon on a node, remove one of them
// 3. If no criteria require for 1 or 2, pick an arbitrary mon
func (c *Cluster) determineExtraMonToRemove() string {
mons := c.clusterInfoToMonConfig()
if c.spec.IsStretchCluster() {
stretchMonToRemove := c.findExtraMonToRemoveFromStretchCluster(mons)
if stretchMonToRemove != "" {
return stretchMonToRemove
}
logger.Infof("did not find an extra mon to remove from the stretch cluster")
return ""
}
nodesWithMons := map[string]string{}
arbitraryMon := ""
for _, mon := range mons {
if mon.NodeName == "" {
logger.Debugf("mon %q is not scheduled to a specific host", mon.DaemonName)
continue
}
// Check if there are multiple mons on the node
if existingMon, ok := nodesWithMons[mon.NodeName]; ok {
logger.Infof("found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName)
return mon.DaemonName
}
nodesWithMons[mon.NodeName] = mon.DaemonName
// assign the current mon as the fallback mon
arbitraryMon = mon.DaemonName
}
logger.Infof("removing arbitrary extra mon %q", arbitraryMon)
return arbitraryMon
}
func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) string {
// Build the count of current mons per zone
zoneCount := map[string]int{}
monInZones := map[string]string{}
for _, m := range mons {
if m.Zone == "" {
logger.Warningf("zone not found on mon %q", m.DaemonName)
continue
}
zoneCount[m.Zone]++
// We just need the name of one of the mons in the zone in case there are extra
monInZones[m.Zone] = m.DaemonName
}
// Find a zone that has too many mons
for _, zone := range c.spec.Mon.StretchCluster.Zones {
count, ok := zoneCount[zone.Name]
if !ok {
// The zone isn't currently assigned to any mon, so skip it
continue
}
if zone.Arbiter {
if count > 1 {
logger.Infof("removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name)
return monInZones[zone.Name]
}
} else {
if count > 2 {
logger.Infof("removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name)
return monInZones[zone.Name]
}
}
}
return ""
}
// failMon compares the monCount against desiredMonCount
// Returns whether the failover request was attempted. If false,
// the operator should check for other mons to failover.
@@ -459,25 +533,8 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
func (c *Cluster) failoverMon(name string) error {
logger.Infof("Failing over monitor %q", name)
// Scale down the failed mon to allow a new one to start
if err := c.updateMonDeploymentReplica(name, false); err != nil {
// attempt to continue with the failover even if the bad mon could not be stopped
logger.Warningf("failed to stop mon %q for failover. %v", name, err)
}
newMonSucceeded := false
defer func() {
if newMonSucceeded {
// do nothing if the new mon was started successfully, the deployment will anyway be deleted
return
}
if err := c.updateMonDeploymentReplica(name, true); err != nil {
// attempt to continue even if the bad mon could not be restarted
logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
}
}()
// remove the failed mon from a local list of the existing mons for finding a stretch zone
existingMons := c.clusterInfoToMonConfig(name)
existingMons := c.clusterInfoToMonConfigWithExclude(name)
zone, err := c.findAvailableZoneIfStretched(existingMons)
if err != nil {
@@ -488,9 +545,40 @@ func (c *Cluster) failoverMon(name string) error {
m := c.newMonConfig(c.maxMonID+1, zone)
logger.Infof("starting new mon: %+v", m)
mConf := []*monConfig{m}
// Scale down the failed mon to allow a new one to start
if err := c.updateMonDeploymentReplica(name, false); err != nil {
// attempt to continue with the failover even if the bad mon could not be stopped
logger.Warningf("failed to stop mon %q for failover. %v", name, err)
}
// If the mon failover is not successful, revert the failover
newMonSucceeded := false
newMonMightBeInQuorum := false
defer func() {
if newMonSucceeded {
// do nothing if the new mon was started successfully, the deployment will anyway be deleted
return
}
logger.Warningf("failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName)
if err := c.updateMonDeploymentReplica(name, true); err != nil {
// attempt to continue even if the bad mon could not be restarted
logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
}
if err := c.removeMonWithOptionalQuorum(m.DaemonName, newMonMightBeInQuorum); err != nil {
logger.Infof("failed to remove mon %q from quorum. %v", m.DaemonName, err)
}
// Make sure the maxMonID is reverted to its previous value
// The maxMonId is committed to a configmap immediately after the mon deployment
// is started, even though c.maxMonID is not incremented until the mon failover is successful
logger.Infof("reverting maxMonId to %d", c.maxMonID)
if err := c.commitMaxMonIDRequireIncrementing(c.maxMonID, false); err != nil {
logger.Errorf("failed to revert maxMonId after starting mon %q", m.DaemonName)
}
}()
// Assign the pod to a node
mConf := []*monConfig{m}
if err := c.assignMons(mConf); err != nil {
return errors.Wrap(err, "failed to place new mon on a node")
}
@@ -512,6 +600,7 @@ func (c *Cluster) failoverMon(name string) error {
c.ClusterInfo.Monitors[m.DaemonName] = cephclient.NewMonInfo(m.DaemonName, m.PublicIP, m.Port)
// Start the deployment
newMonMightBeInQuorum = true
if err := c.startDeployments(mConf, true); err != nil {
return errors.Wrapf(err, "failed to start new mon %s", m.DaemonName)
}
@@ -533,6 +622,15 @@ func (c *Cluster) failoverMon(name string) error {
// make a best effort to remove the mon and all its resources
func (c *Cluster) removeMon(daemonName string) error {
return c.removeMonWithOptionalQuorum(daemonName, true)
}
// make a best effort to remove the mon and all its resources
func (c *Cluster) removeMonWithOptionalQuorum(daemonName string, shouldRemoveFromQuorum bool) error {
if daemonName == "" {
logger.Info("did not identify a mon to remove")
return nil
}
logger.Infof("ensuring removal of unhealthy monitor %s", daemonName)
resourceName := resourceName(daemonName)
@@ -550,8 +648,10 @@ func (c *Cluster) removeMon(daemonName string) error {
}
// Remove the bad monitor from quorum
if err := c.removeMonitorFromQuorum(daemonName); err != nil {
logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err)
if shouldRemoveFromQuorum {
if err := c.removeMonitorFromQuorum(daemonName); err != nil {
logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err)
}
}
delete(c.ClusterInfo.Monitors, daemonName)
@@ -150,6 +150,62 @@ func TestCheckHealth(t *testing.T) {
}
}
func TestRemoveExtraMon(t *testing.T) {
endpoint := "1.2.3.4:6789"
c := &Cluster{mapping: &opcontroller.Mapping{}}
c.ClusterInfo = &cephclient.ClusterInfo{Monitors: map[string]*cephclient.MonInfo{
"a": {Name: "a", Endpoint: endpoint},
"b": {Name: "b", Endpoint: endpoint},
"c": {Name: "c", Endpoint: endpoint},
"d": {Name: "d", Endpoint: endpoint},
}}
c.mapping.Schedule = map[string]*opcontroller.MonScheduleInfo{
"a": {Name: "node1"},
"b": {Name: "node2"},
"c": {Name: "node3"},
"d": {Name: "node1"},
}
// Remove mon an extra mon on the same node
removedMon := c.determineExtraMonToRemove()
if removedMon != "a" && removedMon != "d" {
assert.Fail(t, fmt.Sprintf("removed mon %q instead of a or d", removedMon))
}
// Remove an arbitrary mon that are all on different nodes
c.mapping.Schedule["d"].Name = "node4"
removedMon = c.determineExtraMonToRemove()
assert.NotEqual(t, "", removedMon)
// Don't remove any extra mon from a proper stretch cluster
c.spec.Mon.StretchCluster = &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
{Name: "x", Arbiter: true},
{Name: "y"},
{Name: "z"},
}}
c.ClusterInfo.Monitors["e"] = &cephclient.MonInfo{Name: "e", Endpoint: endpoint}
c.mapping.Schedule["a"].Zone = "x"
c.mapping.Schedule["b"].Zone = "y"
c.mapping.Schedule["c"].Zone = "y"
c.mapping.Schedule["d"].Zone = "z"
c.mapping.Schedule["e"] = &opcontroller.MonScheduleInfo{Name: "node5", Zone: "z"}
removedMon = c.determineExtraMonToRemove()
assert.Equal(t, "", removedMon)
// Remove an extra mon from the arbiter zone
c.mapping.Schedule["d"].Zone = "x"
removedMon = c.determineExtraMonToRemove()
if removedMon != "a" && removedMon != "d" {
assert.Fail(t, "removed mon %q instead of a or d from the arbiter zone", removedMon)
}
// Remove an extra mon from a non-arbiter zone
c.mapping.Schedule["d"].Zone = "y"
removedMon = c.determineExtraMonToRemove()
if removedMon != "b" && removedMon != "c" && removedMon != "d" {
assert.Fail(t, fmt.Sprintf("removed mon %q instead of b, c, or d from the non-arbiter zone", removedMon))
}
}
func TestSkipMonFailover(t *testing.T) {
c := New(context.TODO(), &clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil)
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
+20 -7
View File
@@ -127,6 +127,8 @@ type monConfig struct {
Port int32
// The zone used for a stretch cluster
Zone string
// The node where the mon is assigned
NodeName string
// DataPathMap is the mapping relationship between mon data stored on the host and mon data
// stored in containers.
DataPathMap *config.DataPathMap
@@ -515,7 +517,7 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s
func (c *Cluster) initMonConfig(size int) (int, []*monConfig, error) {
// initialize the mon pod info for mons that have been previously created
mons := c.clusterInfoToMonConfig("")
mons := c.clusterInfoToMonConfig()
// initialize mon info if we don't have enough mons (at first startup)
existingCount := len(c.ClusterInfo.Monitors)
@@ -531,7 +533,11 @@ func (c *Cluster) initMonConfig(size int) (int, []*monConfig, error) {
return existingCount, mons, nil
}
func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig {
func (c *Cluster) clusterInfoToMonConfig() []*monConfig {
return c.clusterInfoToMonConfigWithExclude("")
}
func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monConfig {
mons := []*monConfig{}
for _, monitor := range c.ClusterInfo.Monitors {
if monitor.Name == excludedMon {
@@ -539,9 +545,11 @@ func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig {
continue
}
var zone string
var nodeName string
schedule := c.mapping.Schedule[monitor.Name]
if schedule != nil {
zone = schedule.Zone
nodeName = schedule.Name
}
mons = append(mons, &monConfig{
ResourceName: resourceName(monitor.Name),
@@ -549,6 +557,7 @@ func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig {
Port: cephutil.GetPortFromEndpoint(monitor.Endpoint),
PublicIP: cephutil.GetIPFromEndpoint(monitor.Endpoint),
Zone: zone,
NodeName: nodeName,
DataPathMap: config.NewStatefulDaemonDataPathMap(c.spec.DataDirHostPath, dataDirRelativeHostPath(monitor.Name), config.MonType, monitor.Name, c.Namespace),
})
}
@@ -1125,6 +1134,10 @@ func (c *Cluster) commitMaxMonID(monName string) error {
return errors.Wrapf(err, "invalid mon name %q", monName)
}
return c.commitMaxMonIDRequireIncrementing(committedMonID, true)
}
func (c *Cluster) commitMaxMonIDRequireIncrementing(desiredMaxMonID int, requireIncrementing bool) error {
configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(c.ClusterInfo.Context, EndpointConfigMapName, metav1.GetOptions{})
if err != nil {
return errors.Wrap(err, "failed to find existing mon endpoint config map")
@@ -1136,13 +1149,13 @@ func (c *Cluster) commitMaxMonID(monName string) error {
return errors.Wrap(err, "failed to read existing maxMonId")
}
if existingMax >= committedMonID {
logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", committedMonID, existingMax)
if requireIncrementing && existingMax >= desiredMaxMonID {
logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax)
return nil
}
logger.Infof("updating maxMonID from %d to %d after committing mon %q", existingMax, committedMonID, monName)
configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(committedMonID)
logger.Infof("updating maxMonID from %d to %d", existingMax, desiredMaxMonID)
configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(desiredMaxMonID)
if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "failed to update mon endpoint config map for the maxMonID")
@@ -1327,7 +1340,7 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
// wait for monitors to establish quorum
retryCount := 0
retryMax := 30
retryMax := 60
for {
// Return immediately if the context has been canceled
if clusterInfo.Context.Err() != nil {