mon: remove out of quorum mons from ceph.conf

The mons that are out of quorum may cause ceph commands to timeout
or fail unnecessarily trying to connect to a mon that is no longer
online. Now the mon health check will update the mon endpoints configmap
when a mon is detected out of quorum. This also means that if the
operator is restarted during a mon failover, the failed mon will no
longer remain in the ceph.conf, thus allowing the quorum to be more
likely to respond to the mons that are still in quorum.

The update for mons out of quorum only applies if other mons are in
quorum. If quorum is down, the configmap will not keep track of the offline
mons since too many are offline.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2022-11-08 11:36:37 -07:00
1 parent c67c33f1d3
commit b6e8ea2b50
6 files changed
+168 -13

No files matched your search

+9 -7
View File
@@ -247,15 +247,18 @@ func addClientConfigFileSection(configFile *ini.File, clientName, keyringPath st
// PopulateMonHostMembers extracts a list of just the monitor names, which will populate the "mon initial members"
// and "mon hosts" global config field
func PopulateMonHostMembers(clusterInfo *ClusterInfo) ([]string, []string) {
monMembers := make([]string, len(clusterInfo.Monitors))
monHosts := make([]string, len(clusterInfo.Monitors))
var monMembers []string
var monHosts []string
i := 0
for _, monitor := range clusterInfo.Monitors {
monMembers[i] = monitor.Name
if monitor.OutOfQuorum {
logger.Warningf("skipping adding mon %q to config file, detected out of quorum", monitor.Name)
continue
}
monMembers = append(monMembers, monitor.Name)
monIP := cephutil.GetIPFromEndpoint(monitor.Endpoint)
if clusterInfo.RequireMsgr2 {
monHosts[i] = fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port)
monHosts = append(monHosts, fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port))
} else {
// Detect the current port if the mon already exists
// so the same msgr1 port can be preserved if needed (6789 or 6790)
@@ -266,9 +269,8 @@ func PopulateMonHostMembers(clusterInfo *ClusterInfo) ([]string, []string) {
}
msgr2Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(Msgr2port)))
msgr1Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(currentMonPort)))
monHosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]"
monHosts = append(monHosts, "[v2:"+msgr2Endpoint+",v1:"+msgr1Endpoint+"]")
}
i++
}
return monMembers, monHosts
+2
View File
@@ -63,6 +63,8 @@ type ClusterInfo struct {
type MonInfo struct {
Name string `json:"name"`
Endpoint string `json:"endpoint"`
// Whether detected out of quorum by rook. May be different from actual ceph quorum.
OutOfQuorum bool `json:"outOfQuorum"`
}
// CephCred represents the Ceph cluster username and key used by the operator.
+69 -3
View File
@@ -243,6 +243,9 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
}
if inQuorum {
if _, err := c.trackMonInOrOutOfQuorum(mon.Name, true); err != nil {
return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name)
}
logger.Debugf("mon %q found in quorum", mon.Name)
// delete the "timeout" for a mon if the pod is in quorum again
if _, ok := c.monTimeoutList[mon.Name]; ok {
@@ -253,15 +256,17 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
}
logger.Debugf("mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus)
allMonsInQuorum = false
if _, err := c.trackMonInOrOutOfQuorum(mon.Name, false); err != nil {
return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name)
}
// if the time out is set to 0 this indicate that we don't want to trigger mon failover
if MonOutTimeout == timeZero {
logger.Warningf("mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name)
return nil
continue
}
allMonsInQuorum = false
// If not yet set, add the current time, for the timeout
// calculation, to the list
if _, ok := c.monTimeoutList[mon.Name]; !ok {
@@ -299,6 +304,13 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
return nil
}
if allMonsInQuorum {
// Make sure the mons out of quorum are cleared
if _, err := c.trackMonInOrOutOfQuorum("", true); err != nil {
return errors.Wrap(err, "failed to track all mons in quorum")
}
}
// after all unhealthy mons have been removed or failed over
// handle all mons that haven't been in the Ceph mon map
for mon := range monsNotFound {
@@ -341,6 +353,55 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
return nil
}
func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool, error) {
updateNeeded := false
var monsOutOfQuorum []string
if monName == "" {
// All mons are in quorum, so make sure no mons are marked out of quorum
for monName, mon := range c.ClusterInfo.Monitors {
if mon.OutOfQuorum {
logger.Infof("resetting mon %q to be back in quorum", monName)
mon.OutOfQuorum = false
updateNeeded = true
}
}
} else {
mon, ok := c.ClusterInfo.Monitors[monName]
if !ok {
logger.Infof("mon %q not found to keep track of being out of quorum", monName)
return false, nil
}
// Mark the mon in quorum
if inQuorum && mon.OutOfQuorum {
logger.Infof("marking mon %q back in quorum", monName)
mon.OutOfQuorum = false
updateNeeded = true
}
// Mark the mon out of quorum
if !inQuorum && !mon.OutOfQuorum {
logger.Infof("marking mon %q out of quorum", monName)
mon.OutOfQuorum = true
updateNeeded = true
}
if mon.OutOfQuorum {
monsOutOfQuorum = append(monsOutOfQuorum, monName)
}
}
if updateNeeded {
// write the latest config to the config dir
if err := WriteConnectionConfig(c.context, c.ClusterInfo); err != nil {
return true, errors.Wrap(err, "failed to write connection config for new mons")
}
// Update the mon endpoints configmap
err := controller.UpdateMonsOutOfQuorum(c.context.Clientset, c.Namespace, monsOutOfQuorum)
if err != nil {
return true, errors.Wrap(err, "failed to update mon endpoints cm")
}
}
return updateNeeded, nil
}
// determineExtraMonToRemove assumes all mons are in quorum and that there are more mons
// that required for desired state. One mon will be picked for removal in this priority:
// 1. If a stretch cluster, remove the extra mon according to the stretch topology
@@ -419,6 +480,11 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri
// Returns whether the failover request was attempted. If false,
// the operator should check for other mons to failover.
func (c *Cluster) failMon(monCount, desiredMonCount int, name string) bool {
// make sure the failed mon is marked out of quorum
if _, err := c.trackMonInOrOutOfQuorum(name, false); err != nil {
logger.Errorf("failed to track failed mon %q. %v", name, err)
}
if monCount > desiredMonCount {
// no need to create a new mon since we have an extra
if err := c.removeMon(name); err != nil {
@@ -19,6 +19,7 @@ package mon
import (
"context"
"fmt"
"os"
"reflect"
"testing"
"time"
@@ -28,6 +29,7 @@ import (
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/version"
testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test"
@@ -206,6 +208,51 @@ func TestRemoveExtraMon(t *testing.T) {
}
}
func TestTrackMonsOutOfQuorum(t *testing.T) {
endpoint := "1.2.3.4:6789"
clientset := test.New(t, 1)
tempDir, err := os.MkdirTemp("", "")
assert.NoError(t, err)
defer os.RemoveAll(tempDir)
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
c := &Cluster{
mapping: &opcontroller.Mapping{},
context: &clusterd.Context{Clientset: clientset, ConfigDir: tempDir},
ownerInfo: ownerInfo,
Namespace: "ns"}
c.ClusterInfo = &cephclient.ClusterInfo{Monitors: map[string]*cephclient.MonInfo{
"a": {Name: "a", Endpoint: endpoint},
"b": {Name: "b", Endpoint: endpoint},
"c": {Name: "c", Endpoint: endpoint},
}}
// No change since all mons are in quorum
updated, err := c.trackMonInOrOutOfQuorum("a", true)
assert.False(t, updated)
assert.NoError(t, err)
// initialize the configmap
err = c.persistExpectedMonDaemons()
assert.NoError(t, err)
// Track mon.a as out of quorum
updated, err = c.trackMonInOrOutOfQuorum("a", false)
assert.True(t, updated)
assert.NoError(t, err)
cm, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
assert.NoError(t, err)
assert.Equal(t, "a", cm.Data[controller.OutOfQuorumKey])
// Put mon.a back in quorum
updated, err = c.trackMonInOrOutOfQuorum("a", true)
assert.True(t, updated)
assert.NoError(t, err)
cm, err = clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
assert.NoError(t, err)
assert.Equal(t, "", cm.Data[controller.OutOfQuorumKey])
}
func TestSkipMonFailover(t *testing.T) {
c := New(context.TODO(), &clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil)
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
+12 -3
View File
@@ -1097,6 +1097,14 @@ func (c *Cluster) persistExpectedMonDaemons() error {
return errors.Wrap(err, "failed to save maxMonID")
}
// preserve the mons detected out of quorum
var monsOutOfQuorum []string
for monName, mon := range c.ClusterInfo.Monitors {
if mon.OutOfQuorum {
monsOutOfQuorum = append(monsOutOfQuorum, monName)
}
}
configMap.Data = map[string]string{
EndpointDataKey: FlattenMonEndpoints(c.ClusterInfo.Monitors),
// persist the maxMonID that was previously stored in the configmap. We are likely saving info
@@ -1104,9 +1112,10 @@ func (c *Cluster) persistExpectedMonDaemons() error {
// actually been started. If the operator is restarted or the reconcile is otherwise restarted,
// we want to calculate the mon scheduling next time based on the committed maxMonID, rather
// than only a mon scheduling, which may not have completed.
controller.MaxMonIDKey: maxMonID,
controller.MappingKey: string(monMapping),
csi.ConfigKey: csiConfigValue,
controller.MaxMonIDKey: maxMonID,
controller.MappingKey: string(monMapping),
controller.OutOfQuorumKey: strings.Join(monsOutOfQuorum, ","),
csi.ConfigKey: csiConfigValue,
}
if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(c.ClusterInfo.Context, configMap, metav1.CreateOptions{}); err != nil {
@@ -56,6 +56,8 @@ const (
EndpointConfigMapName = "rook-ceph-mon-endpoints"
// EndpointDataKey is the name of the key inside the mon configmap to get the endpoints
EndpointDataKey = "data"
// OutOfQuorumKey is the name of the key for tracking mons detected out of quorum
OutOfQuorumKey = "outOfQuorum"
// MaxMonIDKey is the name of the max mon id used
MaxMonIDKey = "maxMonId"
// MappingKey is the name of the mapping for the mon->node and node->port
@@ -249,6 +251,21 @@ func ExtractKey(contents string) (string, error) {
return secret, nil
}
func UpdateMonsOutOfQuorum(clientset kubernetes.Interface, namespace string, monsOutOfQuorum []string) error {
ctx := context.TODO()
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{})
if err != nil {
return errors.Wrap(err, "failed to get mon endpoints configmap")
}
cm.Data[OutOfQuorumKey] = strings.Join(monsOutOfQuorum, ",")
_, err = clientset.CoreV1().ConfigMaps(namespace).Update(ctx, cm, metav1.UpdateOptions{})
if err != nil {
return errors.Wrap(err, "failed to update mon endpoints configmap with mon(s) out of quorum")
}
return nil
}
// loadMonConfig returns the monitor endpoints and maxMonID
func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string]*cephclient.MonInfo, int, *Mapping, error) {
ctx := context.TODO()
@@ -272,6 +289,18 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string
monEndpointMap = ParseMonEndpoints(info)
}
// Parse the mons that were detected out of quorum
if outOfQuorum, ok := cm.Data[OutOfQuorumKey]; ok && len(outOfQuorum) > 0 {
monNames := strings.Split(outOfQuorum, ",")
for _, monName := range monNames {
if monInfo, ok := monEndpointMap[monName]; ok {
monInfo.OutOfQuorum = true
} else {
logger.Warningf("did not find mon %q to set it out of quorum in the cluster info", monName)
}
}
}
// Parse the max monitor id
storedMaxMonID := -1
if id, ok := cm.Data[MaxMonIDKey]; ok {