diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index d9573189c..4393cc54d 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -320,7 +320,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error { logger.Warningf("cannot reduce mon quorum size from 2 to 1") } else { logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount) - return c.removeMon(quorumStatus.MonMap.Mons[0].Name) + return c.removeMon(c.determineExtraMonToRemove()) } } @@ -341,6 +341,80 @@ func (c *Cluster) checkHealth(ctx context.Context) error { return nil } +// determineExtraMonToRemove assumes all mons are in quorum and that there are more mons +// that required for desired state. One mon will be picked for removal in this priority: +// 1. If a stretch cluster, remove the extra mon according to the stretch topology +// 2. If more than one mon on a node, remove one of them +// 3. If no criteria require for 1 or 2, pick an arbitrary mon +func (c *Cluster) determineExtraMonToRemove() string { + mons := c.clusterInfoToMonConfig() + if c.spec.IsStretchCluster() { + stretchMonToRemove := c.findExtraMonToRemoveFromStretchCluster(mons) + if stretchMonToRemove != "" { + return stretchMonToRemove + } + logger.Infof("did not find an extra mon to remove from the stretch cluster") + return "" + } + + nodesWithMons := map[string]string{} + arbitraryMon := "" + for _, mon := range mons { + if mon.NodeName == "" { + logger.Debugf("mon %q is not scheduled to a specific host", mon.DaemonName) + continue + } + // Check if there are multiple mons on the node + if existingMon, ok := nodesWithMons[mon.NodeName]; ok { + logger.Infof("found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName) + return mon.DaemonName + } + nodesWithMons[mon.NodeName] = mon.DaemonName + + // assign the current mon as the fallback mon + arbitraryMon = mon.DaemonName + } + + logger.Infof("removing arbitrary extra mon %q", arbitraryMon) + return arbitraryMon +} + +func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) string { + // Build the count of current mons per zone + zoneCount := map[string]int{} + monInZones := map[string]string{} + for _, m := range mons { + if m.Zone == "" { + logger.Warningf("zone not found on mon %q", m.DaemonName) + continue + } + zoneCount[m.Zone]++ + // We just need the name of one of the mons in the zone in case there are extra + monInZones[m.Zone] = m.DaemonName + } + + // Find a zone that has too many mons + for _, zone := range c.spec.Mon.StretchCluster.Zones { + count, ok := zoneCount[zone.Name] + if !ok { + // The zone isn't currently assigned to any mon, so skip it + continue + } + if zone.Arbiter { + if count > 1 { + logger.Infof("removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name) + return monInZones[zone.Name] + } + } else { + if count > 2 { + logger.Infof("removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name) + return monInZones[zone.Name] + } + } + } + return "" +} + // failMon compares the monCount against desiredMonCount // Returns whether the failover request was attempted. If false, // the operator should check for other mons to failover. @@ -459,25 +533,8 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error { func (c *Cluster) failoverMon(name string) error { logger.Infof("Failing over monitor %q", name) - // Scale down the failed mon to allow a new one to start - if err := c.updateMonDeploymentReplica(name, false); err != nil { - // attempt to continue with the failover even if the bad mon could not be stopped - logger.Warningf("failed to stop mon %q for failover. %v", name, err) - } - newMonSucceeded := false - defer func() { - if newMonSucceeded { - // do nothing if the new mon was started successfully, the deployment will anyway be deleted - return - } - if err := c.updateMonDeploymentReplica(name, true); err != nil { - // attempt to continue even if the bad mon could not be restarted - logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err) - } - }() - // remove the failed mon from a local list of the existing mons for finding a stretch zone - existingMons := c.clusterInfoToMonConfig(name) + existingMons := c.clusterInfoToMonConfigWithExclude(name) zone, err := c.findAvailableZoneIfStretched(existingMons) if err != nil { @@ -488,9 +545,40 @@ func (c *Cluster) failoverMon(name string) error { m := c.newMonConfig(c.maxMonID+1, zone) logger.Infof("starting new mon: %+v", m) - mConf := []*monConfig{m} + // Scale down the failed mon to allow a new one to start + if err := c.updateMonDeploymentReplica(name, false); err != nil { + // attempt to continue with the failover even if the bad mon could not be stopped + logger.Warningf("failed to stop mon %q for failover. %v", name, err) + } + + // If the mon failover is not successful, revert the failover + newMonSucceeded := false + newMonMightBeInQuorum := false + defer func() { + if newMonSucceeded { + // do nothing if the new mon was started successfully, the deployment will anyway be deleted + return + } + logger.Warningf("failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName) + if err := c.updateMonDeploymentReplica(name, true); err != nil { + // attempt to continue even if the bad mon could not be restarted + logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err) + } + if err := c.removeMonWithOptionalQuorum(m.DaemonName, newMonMightBeInQuorum); err != nil { + logger.Infof("failed to remove mon %q from quorum. %v", m.DaemonName, err) + } + + // Make sure the maxMonID is reverted to its previous value + // The maxMonId is committed to a configmap immediately after the mon deployment + // is started, even though c.maxMonID is not incremented until the mon failover is successful + logger.Infof("reverting maxMonId to %d", c.maxMonID) + if err := c.commitMaxMonIDRequireIncrementing(c.maxMonID, false); err != nil { + logger.Errorf("failed to revert maxMonId after starting mon %q", m.DaemonName) + } + }() // Assign the pod to a node + mConf := []*monConfig{m} if err := c.assignMons(mConf); err != nil { return errors.Wrap(err, "failed to place new mon on a node") } @@ -512,6 +600,7 @@ func (c *Cluster) failoverMon(name string) error { c.ClusterInfo.Monitors[m.DaemonName] = cephclient.NewMonInfo(m.DaemonName, m.PublicIP, m.Port) // Start the deployment + newMonMightBeInQuorum = true if err := c.startDeployments(mConf, true); err != nil { return errors.Wrapf(err, "failed to start new mon %s", m.DaemonName) } @@ -533,6 +622,15 @@ func (c *Cluster) failoverMon(name string) error { // make a best effort to remove the mon and all its resources func (c *Cluster) removeMon(daemonName string) error { + return c.removeMonWithOptionalQuorum(daemonName, true) +} + +// make a best effort to remove the mon and all its resources +func (c *Cluster) removeMonWithOptionalQuorum(daemonName string, shouldRemoveFromQuorum bool) error { + if daemonName == "" { + logger.Info("did not identify a mon to remove") + return nil + } logger.Infof("ensuring removal of unhealthy monitor %s", daemonName) resourceName := resourceName(daemonName) @@ -550,8 +648,10 @@ func (c *Cluster) removeMon(daemonName string) error { } // Remove the bad monitor from quorum - if err := c.removeMonitorFromQuorum(daemonName); err != nil { - logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err) + if shouldRemoveFromQuorum { + if err := c.removeMonitorFromQuorum(daemonName); err != nil { + logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err) + } } delete(c.ClusterInfo.Monitors, daemonName) diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 2d6c70b26..25131b628 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -150,6 +150,62 @@ func TestCheckHealth(t *testing.T) { } } +func TestRemoveExtraMon(t *testing.T) { + endpoint := "1.2.3.4:6789" + c := &Cluster{mapping: &opcontroller.Mapping{}} + c.ClusterInfo = &cephclient.ClusterInfo{Monitors: map[string]*cephclient.MonInfo{ + "a": {Name: "a", Endpoint: endpoint}, + "b": {Name: "b", Endpoint: endpoint}, + "c": {Name: "c", Endpoint: endpoint}, + "d": {Name: "d", Endpoint: endpoint}, + }} + c.mapping.Schedule = map[string]*opcontroller.MonScheduleInfo{ + "a": {Name: "node1"}, + "b": {Name: "node2"}, + "c": {Name: "node3"}, + "d": {Name: "node1"}, + } + // Remove mon an extra mon on the same node + removedMon := c.determineExtraMonToRemove() + if removedMon != "a" && removedMon != "d" { + assert.Fail(t, fmt.Sprintf("removed mon %q instead of a or d", removedMon)) + } + + // Remove an arbitrary mon that are all on different nodes + c.mapping.Schedule["d"].Name = "node4" + removedMon = c.determineExtraMonToRemove() + assert.NotEqual(t, "", removedMon) + + // Don't remove any extra mon from a proper stretch cluster + c.spec.Mon.StretchCluster = &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ + {Name: "x", Arbiter: true}, + {Name: "y"}, + {Name: "z"}, + }} + c.ClusterInfo.Monitors["e"] = &cephclient.MonInfo{Name: "e", Endpoint: endpoint} + c.mapping.Schedule["a"].Zone = "x" + c.mapping.Schedule["b"].Zone = "y" + c.mapping.Schedule["c"].Zone = "y" + c.mapping.Schedule["d"].Zone = "z" + c.mapping.Schedule["e"] = &opcontroller.MonScheduleInfo{Name: "node5", Zone: "z"} + removedMon = c.determineExtraMonToRemove() + assert.Equal(t, "", removedMon) + + // Remove an extra mon from the arbiter zone + c.mapping.Schedule["d"].Zone = "x" + removedMon = c.determineExtraMonToRemove() + if removedMon != "a" && removedMon != "d" { + assert.Fail(t, "removed mon %q instead of a or d from the arbiter zone", removedMon) + } + + // Remove an extra mon from a non-arbiter zone + c.mapping.Schedule["d"].Zone = "y" + removedMon = c.determineExtraMonToRemove() + if removedMon != "b" && removedMon != "c" && removedMon != "d" { + assert.Fail(t, fmt.Sprintf("removed mon %q instead of b, c, or d from the non-arbiter zone", removedMon)) + } +} + func TestSkipMonFailover(t *testing.T) { c := New(context.TODO(), &clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil) c.ClusterInfo = clienttest.CreateTestClusterInfo(1) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 7d7ade30d..0ad758ce8 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -127,6 +127,8 @@ type monConfig struct { Port int32 // The zone used for a stretch cluster Zone string + // The node where the mon is assigned + NodeName string // DataPathMap is the mapping relationship between mon data stored on the host and mon data // stored in containers. DataPathMap *config.DataPathMap @@ -515,7 +517,7 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s func (c *Cluster) initMonConfig(size int) (int, []*monConfig, error) { // initialize the mon pod info for mons that have been previously created - mons := c.clusterInfoToMonConfig("") + mons := c.clusterInfoToMonConfig() // initialize mon info if we don't have enough mons (at first startup) existingCount := len(c.ClusterInfo.Monitors) @@ -531,7 +533,11 @@ func (c *Cluster) initMonConfig(size int) (int, []*monConfig, error) { return existingCount, mons, nil } -func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig { +func (c *Cluster) clusterInfoToMonConfig() []*monConfig { + return c.clusterInfoToMonConfigWithExclude("") +} + +func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monConfig { mons := []*monConfig{} for _, monitor := range c.ClusterInfo.Monitors { if monitor.Name == excludedMon { @@ -539,9 +545,11 @@ func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig { continue } var zone string + var nodeName string schedule := c.mapping.Schedule[monitor.Name] if schedule != nil { zone = schedule.Zone + nodeName = schedule.Name } mons = append(mons, &monConfig{ ResourceName: resourceName(monitor.Name), @@ -549,6 +557,7 @@ func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig { Port: cephutil.GetPortFromEndpoint(monitor.Endpoint), PublicIP: cephutil.GetIPFromEndpoint(monitor.Endpoint), Zone: zone, + NodeName: nodeName, DataPathMap: config.NewStatefulDaemonDataPathMap(c.spec.DataDirHostPath, dataDirRelativeHostPath(monitor.Name), config.MonType, monitor.Name, c.Namespace), }) } @@ -1125,6 +1134,10 @@ func (c *Cluster) commitMaxMonID(monName string) error { return errors.Wrapf(err, "invalid mon name %q", monName) } + return c.commitMaxMonIDRequireIncrementing(committedMonID, true) +} + +func (c *Cluster) commitMaxMonIDRequireIncrementing(desiredMaxMonID int, requireIncrementing bool) error { configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(c.ClusterInfo.Context, EndpointConfigMapName, metav1.GetOptions{}) if err != nil { return errors.Wrap(err, "failed to find existing mon endpoint config map") @@ -1136,13 +1149,13 @@ func (c *Cluster) commitMaxMonID(monName string) error { return errors.Wrap(err, "failed to read existing maxMonId") } - if existingMax >= committedMonID { - logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", committedMonID, existingMax) + if requireIncrementing && existingMax >= desiredMaxMonID { + logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax) return nil } - logger.Infof("updating maxMonID from %d to %d after committing mon %q", existingMax, committedMonID, monName) - configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(committedMonID) + logger.Infof("updating maxMonID from %d to %d", existingMax, desiredMaxMonID) + configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(desiredMaxMonID) if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil { return errors.Wrap(err, "failed to update mon endpoint config map for the maxMonID") @@ -1327,7 +1340,7 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl // wait for monitors to establish quorum retryCount := 0 - retryMax := 30 + retryMax := 60 for { // Return immediately if the context has been canceled if clusterInfo.Context.Err() != nil {