diff --git a/pkg/daemon/ceph/client/config.go b/pkg/daemon/ceph/client/config.go index ea0c594b5..5295230d8 100644 --- a/pkg/daemon/ceph/client/config.go +++ b/pkg/daemon/ceph/client/config.go @@ -247,15 +247,18 @@ func addClientConfigFileSection(configFile *ini.File, clientName, keyringPath st // PopulateMonHostMembers extracts a list of just the monitor names, which will populate the "mon initial members" // and "mon hosts" global config field func PopulateMonHostMembers(clusterInfo *ClusterInfo) ([]string, []string) { - monMembers := make([]string, len(clusterInfo.Monitors)) - monHosts := make([]string, len(clusterInfo.Monitors)) + var monMembers []string + var monHosts []string - i := 0 for _, monitor := range clusterInfo.Monitors { - monMembers[i] = monitor.Name + if monitor.OutOfQuorum { + logger.Warningf("skipping adding mon %q to config file, detected out of quorum", monitor.Name) + continue + } + monMembers = append(monMembers, monitor.Name) monIP := cephutil.GetIPFromEndpoint(monitor.Endpoint) if clusterInfo.RequireMsgr2 { - monHosts[i] = fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port) + monHosts = append(monHosts, fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port)) } else { // Detect the current port if the mon already exists // so the same msgr1 port can be preserved if needed (6789 or 6790) @@ -266,9 +269,8 @@ func PopulateMonHostMembers(clusterInfo *ClusterInfo) ([]string, []string) { } msgr2Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(Msgr2port))) msgr1Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(currentMonPort))) - monHosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]" + monHosts = append(monHosts, "[v2:"+msgr2Endpoint+",v1:"+msgr1Endpoint+"]") } - i++ } return monMembers, monHosts diff --git a/pkg/daemon/ceph/client/info.go b/pkg/daemon/ceph/client/info.go index ea457bf42..d192e10a7 100644 --- a/pkg/daemon/ceph/client/info.go +++ b/pkg/daemon/ceph/client/info.go @@ -63,6 +63,8 @@ type ClusterInfo struct { type MonInfo struct { Name string `json:"name"` Endpoint string `json:"endpoint"` + // Whether detected out of quorum by rook. May be different from actual ceph quorum. + OutOfQuorum bool `json:"outOfQuorum"` } // CephCred represents the Ceph cluster username and key used by the operator. diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index a31f173ca..2e0a40e05 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -243,6 +243,9 @@ func (c *Cluster) checkHealth(ctx context.Context) error { } if inQuorum { + if _, err := c.trackMonInOrOutOfQuorum(mon.Name, true); err != nil { + return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name) + } logger.Debugf("mon %q found in quorum", mon.Name) // delete the "timeout" for a mon if the pod is in quorum again if _, ok := c.monTimeoutList[mon.Name]; ok { @@ -253,15 +256,17 @@ func (c *Cluster) checkHealth(ctx context.Context) error { } logger.Debugf("mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus) + allMonsInQuorum = false + if _, err := c.trackMonInOrOutOfQuorum(mon.Name, false); err != nil { + return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name) + } // if the time out is set to 0 this indicate that we don't want to trigger mon failover if MonOutTimeout == timeZero { logger.Warningf("mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name) - return nil + continue } - allMonsInQuorum = false - // If not yet set, add the current time, for the timeout // calculation, to the list if _, ok := c.monTimeoutList[mon.Name]; !ok { @@ -299,6 +304,13 @@ func (c *Cluster) checkHealth(ctx context.Context) error { return nil } + if allMonsInQuorum { + // Make sure the mons out of quorum are cleared + if _, err := c.trackMonInOrOutOfQuorum("", true); err != nil { + return errors.Wrap(err, "failed to track all mons in quorum") + } + } + // after all unhealthy mons have been removed or failed over // handle all mons that haven't been in the Ceph mon map for mon := range monsNotFound { @@ -341,6 +353,55 @@ func (c *Cluster) checkHealth(ctx context.Context) error { return nil } +func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool, error) { + updateNeeded := false + var monsOutOfQuorum []string + if monName == "" { + // All mons are in quorum, so make sure no mons are marked out of quorum + for monName, mon := range c.ClusterInfo.Monitors { + if mon.OutOfQuorum { + logger.Infof("resetting mon %q to be back in quorum", monName) + mon.OutOfQuorum = false + updateNeeded = true + } + } + } else { + mon, ok := c.ClusterInfo.Monitors[monName] + if !ok { + logger.Infof("mon %q not found to keep track of being out of quorum", monName) + return false, nil + } + // Mark the mon in quorum + if inQuorum && mon.OutOfQuorum { + logger.Infof("marking mon %q back in quorum", monName) + mon.OutOfQuorum = false + updateNeeded = true + } + // Mark the mon out of quorum + if !inQuorum && !mon.OutOfQuorum { + logger.Infof("marking mon %q out of quorum", monName) + mon.OutOfQuorum = true + updateNeeded = true + } + if mon.OutOfQuorum { + monsOutOfQuorum = append(monsOutOfQuorum, monName) + } + } + if updateNeeded { + // write the latest config to the config dir + if err := WriteConnectionConfig(c.context, c.ClusterInfo); err != nil { + return true, errors.Wrap(err, "failed to write connection config for new mons") + } + // Update the mon endpoints configmap + err := controller.UpdateMonsOutOfQuorum(c.context.Clientset, c.Namespace, monsOutOfQuorum) + if err != nil { + return true, errors.Wrap(err, "failed to update mon endpoints cm") + } + } + + return updateNeeded, nil +} + // determineExtraMonToRemove assumes all mons are in quorum and that there are more mons // that required for desired state. One mon will be picked for removal in this priority: // 1. If a stretch cluster, remove the extra mon according to the stretch topology @@ -419,6 +480,11 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri // Returns whether the failover request was attempted. If false, // the operator should check for other mons to failover. func (c *Cluster) failMon(monCount, desiredMonCount int, name string) bool { + // make sure the failed mon is marked out of quorum + if _, err := c.trackMonInOrOutOfQuorum(name, false); err != nil { + logger.Errorf("failed to track failed mon %q. %v", name, err) + } + if monCount > desiredMonCount { // no need to create a new mon since we have an extra if err := c.removeMon(name); err != nil { diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 25131b628..c930c21d8 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -19,6 +19,7 @@ package mon import ( "context" "fmt" + "os" "reflect" "testing" "time" @@ -28,6 +29,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/controller" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/version" testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test" @@ -206,6 +208,51 @@ func TestRemoveExtraMon(t *testing.T) { } } +func TestTrackMonsOutOfQuorum(t *testing.T) { + endpoint := "1.2.3.4:6789" + clientset := test.New(t, 1) + tempDir, err := os.MkdirTemp("", "") + assert.NoError(t, err) + defer os.RemoveAll(tempDir) + ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() + c := &Cluster{ + mapping: &opcontroller.Mapping{}, + context: &clusterd.Context{Clientset: clientset, ConfigDir: tempDir}, + ownerInfo: ownerInfo, + Namespace: "ns"} + c.ClusterInfo = &cephclient.ClusterInfo{Monitors: map[string]*cephclient.MonInfo{ + "a": {Name: "a", Endpoint: endpoint}, + "b": {Name: "b", Endpoint: endpoint}, + "c": {Name: "c", Endpoint: endpoint}, + }} + // No change since all mons are in quorum + updated, err := c.trackMonInOrOutOfQuorum("a", true) + assert.False(t, updated) + assert.NoError(t, err) + + // initialize the configmap + err = c.persistExpectedMonDaemons() + assert.NoError(t, err) + + // Track mon.a as out of quorum + updated, err = c.trackMonInOrOutOfQuorum("a", false) + assert.True(t, updated) + assert.NoError(t, err) + + cm, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, "a", cm.Data[controller.OutOfQuorumKey]) + + // Put mon.a back in quorum + updated, err = c.trackMonInOrOutOfQuorum("a", true) + assert.True(t, updated) + assert.NoError(t, err) + + cm, err = clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, "", cm.Data[controller.OutOfQuorumKey]) +} + func TestSkipMonFailover(t *testing.T) { c := New(context.TODO(), &clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil) c.ClusterInfo = clienttest.CreateTestClusterInfo(1) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 9fa2dc34a..4c6d1860e 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -1097,6 +1097,14 @@ func (c *Cluster) persistExpectedMonDaemons() error { return errors.Wrap(err, "failed to save maxMonID") } + // preserve the mons detected out of quorum + var monsOutOfQuorum []string + for monName, mon := range c.ClusterInfo.Monitors { + if mon.OutOfQuorum { + monsOutOfQuorum = append(monsOutOfQuorum, monName) + } + } + configMap.Data = map[string]string{ EndpointDataKey: FlattenMonEndpoints(c.ClusterInfo.Monitors), // persist the maxMonID that was previously stored in the configmap. We are likely saving info @@ -1104,9 +1112,10 @@ func (c *Cluster) persistExpectedMonDaemons() error { // actually been started. If the operator is restarted or the reconcile is otherwise restarted, // we want to calculate the mon scheduling next time based on the committed maxMonID, rather // than only a mon scheduling, which may not have completed. - controller.MaxMonIDKey: maxMonID, - controller.MappingKey: string(monMapping), - csi.ConfigKey: csiConfigValue, + controller.MaxMonIDKey: maxMonID, + controller.MappingKey: string(monMapping), + controller.OutOfQuorumKey: strings.Join(monsOutOfQuorum, ","), + csi.ConfigKey: csiConfigValue, } if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(c.ClusterInfo.Context, configMap, metav1.CreateOptions{}); err != nil { diff --git a/pkg/operator/ceph/controller/cluster_info.go b/pkg/operator/ceph/controller/cluster_info.go index 4078bb9e8..d801b3039 100644 --- a/pkg/operator/ceph/controller/cluster_info.go +++ b/pkg/operator/ceph/controller/cluster_info.go @@ -56,6 +56,8 @@ const ( EndpointConfigMapName = "rook-ceph-mon-endpoints" // EndpointDataKey is the name of the key inside the mon configmap to get the endpoints EndpointDataKey = "data" + // OutOfQuorumKey is the name of the key for tracking mons detected out of quorum + OutOfQuorumKey = "outOfQuorum" // MaxMonIDKey is the name of the max mon id used MaxMonIDKey = "maxMonId" // MappingKey is the name of the mapping for the mon->node and node->port @@ -249,6 +251,21 @@ func ExtractKey(contents string) (string, error) { return secret, nil } +func UpdateMonsOutOfQuorum(clientset kubernetes.Interface, namespace string, monsOutOfQuorum []string) error { + ctx := context.TODO() + cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "failed to get mon endpoints configmap") + } + + cm.Data[OutOfQuorumKey] = strings.Join(monsOutOfQuorum, ",") + _, err = clientset.CoreV1().ConfigMaps(namespace).Update(ctx, cm, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "failed to update mon endpoints configmap with mon(s) out of quorum") + } + return nil +} + // loadMonConfig returns the monitor endpoints and maxMonID func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string]*cephclient.MonInfo, int, *Mapping, error) { ctx := context.TODO() @@ -272,6 +289,18 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string monEndpointMap = ParseMonEndpoints(info) } + // Parse the mons that were detected out of quorum + if outOfQuorum, ok := cm.Data[OutOfQuorumKey]; ok && len(outOfQuorum) > 0 { + monNames := strings.Split(outOfQuorum, ",") + for _, monName := range monNames { + if monInfo, ok := monEndpointMap[monName]; ok { + monInfo.OutOfQuorum = true + } else { + logger.Warningf("did not find mon %q to set it out of quorum in the cluster info", monName) + } + } + } + // Parse the max monitor id storedMaxMonID := -1 if id, ok := cm.Data[MaxMonIDKey]; ok {