mon: remove out of quorum mons from ceph.conf
The mons that are out of quorum may cause ceph commands to timeout or fail unnecessarily trying to connect to a mon that is no longer online. Now the mon health check will update the mon endpoints configmap when a mon is detected out of quorum. This also means that if the operator is restarted during a mon failover, the failed mon will no longer remain in the ceph.conf, thus allowing the quorum to be more likely to respond to the mons that are still in quorum. The update for mons out of quorum only applies if other mons are in quorum. If quorum is down, the configmap will not keep track of the offline mons since too many are offline. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
6 files changed
+168
-13
No files matched your search
@@ -247,15 +247,18 @@ func addClientConfigFileSection(configFile *ini.File, clientName, keyringPath st
|
||||
// PopulateMonHostMembers extracts a list of just the monitor names, which will populate the "mon initial members"
|
||||
// and "mon hosts" global config field
|
||||
func PopulateMonHostMembers(clusterInfo *ClusterInfo) ([]string, []string) {
|
||||
monMembers := make([]string, len(clusterInfo.Monitors))
|
||||
monHosts := make([]string, len(clusterInfo.Monitors))
|
||||
var monMembers []string
|
||||
var monHosts []string
|
||||
|
||||
i := 0
|
||||
for _, monitor := range clusterInfo.Monitors {
|
||||
monMembers[i] = monitor.Name
|
||||
if monitor.OutOfQuorum {
|
||||
logger.Warningf("skipping adding mon %q to config file, detected out of quorum", monitor.Name)
|
||||
continue
|
||||
}
|
||||
monMembers = append(monMembers, monitor.Name)
|
||||
monIP := cephutil.GetIPFromEndpoint(monitor.Endpoint)
|
||||
if clusterInfo.RequireMsgr2 {
|
||||
monHosts[i] = fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port)
|
||||
monHosts = append(monHosts, fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port))
|
||||
} else {
|
||||
// Detect the current port if the mon already exists
|
||||
// so the same msgr1 port can be preserved if needed (6789 or 6790)
|
||||
@@ -266,9 +269,8 @@ func PopulateMonHostMembers(clusterInfo *ClusterInfo) ([]string, []string) {
|
||||
}
|
||||
msgr2Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(Msgr2port)))
|
||||
msgr1Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(currentMonPort)))
|
||||
monHosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]"
|
||||
monHosts = append(monHosts, "[v2:"+msgr2Endpoint+",v1:"+msgr1Endpoint+"]")
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
return monMembers, monHosts
|
||||
|
||||
@@ -63,6 +63,8 @@ type ClusterInfo struct {
|
||||
type MonInfo struct {
|
||||
Name string `json:"name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
// Whether detected out of quorum by rook. May be different from actual ceph quorum.
|
||||
OutOfQuorum bool `json:"outOfQuorum"`
|
||||
}
|
||||
|
||||
// CephCred represents the Ceph cluster username and key used by the operator.
|
||||
|
||||
@@ -243,6 +243,9 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
}
|
||||
|
||||
if inQuorum {
|
||||
if _, err := c.trackMonInOrOutOfQuorum(mon.Name, true); err != nil {
|
||||
return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name)
|
||||
}
|
||||
logger.Debugf("mon %q found in quorum", mon.Name)
|
||||
// delete the "timeout" for a mon if the pod is in quorum again
|
||||
if _, ok := c.monTimeoutList[mon.Name]; ok {
|
||||
@@ -253,15 +256,17 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
}
|
||||
|
||||
logger.Debugf("mon %q NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus)
|
||||
allMonsInQuorum = false
|
||||
if _, err := c.trackMonInOrOutOfQuorum(mon.Name, false); err != nil {
|
||||
return errors.Wrapf(err, "failed to track out of quorum mon %q", mon.Name)
|
||||
}
|
||||
|
||||
// if the time out is set to 0 this indicate that we don't want to trigger mon failover
|
||||
if MonOutTimeout == timeZero {
|
||||
logger.Warningf("mon %q NOT found in quorum and health timeout is 0, mon will never fail over", mon.Name)
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
|
||||
allMonsInQuorum = false
|
||||
|
||||
// If not yet set, add the current time, for the timeout
|
||||
// calculation, to the list
|
||||
if _, ok := c.monTimeoutList[mon.Name]; !ok {
|
||||
@@ -299,6 +304,13 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if allMonsInQuorum {
|
||||
// Make sure the mons out of quorum are cleared
|
||||
if _, err := c.trackMonInOrOutOfQuorum("", true); err != nil {
|
||||
return errors.Wrap(err, "failed to track all mons in quorum")
|
||||
}
|
||||
}
|
||||
|
||||
// after all unhealthy mons have been removed or failed over
|
||||
// handle all mons that haven't been in the Ceph mon map
|
||||
for mon := range monsNotFound {
|
||||
@@ -341,6 +353,55 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) trackMonInOrOutOfQuorum(monName string, inQuorum bool) (bool, error) {
|
||||
updateNeeded := false
|
||||
var monsOutOfQuorum []string
|
||||
if monName == "" {
|
||||
// All mons are in quorum, so make sure no mons are marked out of quorum
|
||||
for monName, mon := range c.ClusterInfo.Monitors {
|
||||
if mon.OutOfQuorum {
|
||||
logger.Infof("resetting mon %q to be back in quorum", monName)
|
||||
mon.OutOfQuorum = false
|
||||
updateNeeded = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
mon, ok := c.ClusterInfo.Monitors[monName]
|
||||
if !ok {
|
||||
logger.Infof("mon %q not found to keep track of being out of quorum", monName)
|
||||
return false, nil
|
||||
}
|
||||
// Mark the mon in quorum
|
||||
if inQuorum && mon.OutOfQuorum {
|
||||
logger.Infof("marking mon %q back in quorum", monName)
|
||||
mon.OutOfQuorum = false
|
||||
updateNeeded = true
|
||||
}
|
||||
// Mark the mon out of quorum
|
||||
if !inQuorum && !mon.OutOfQuorum {
|
||||
logger.Infof("marking mon %q out of quorum", monName)
|
||||
mon.OutOfQuorum = true
|
||||
updateNeeded = true
|
||||
}
|
||||
if mon.OutOfQuorum {
|
||||
monsOutOfQuorum = append(monsOutOfQuorum, monName)
|
||||
}
|
||||
}
|
||||
if updateNeeded {
|
||||
// write the latest config to the config dir
|
||||
if err := WriteConnectionConfig(c.context, c.ClusterInfo); err != nil {
|
||||
return true, errors.Wrap(err, "failed to write connection config for new mons")
|
||||
}
|
||||
// Update the mon endpoints configmap
|
||||
err := controller.UpdateMonsOutOfQuorum(c.context.Clientset, c.Namespace, monsOutOfQuorum)
|
||||
if err != nil {
|
||||
return true, errors.Wrap(err, "failed to update mon endpoints cm")
|
||||
}
|
||||
}
|
||||
|
||||
return updateNeeded, nil
|
||||
}
|
||||
|
||||
// determineExtraMonToRemove assumes all mons are in quorum and that there are more mons
|
||||
// that required for desired state. One mon will be picked for removal in this priority:
|
||||
// 1. If a stretch cluster, remove the extra mon according to the stretch topology
|
||||
@@ -419,6 +480,11 @@ func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) stri
|
||||
// Returns whether the failover request was attempted. If false,
|
||||
// the operator should check for other mons to failover.
|
||||
func (c *Cluster) failMon(monCount, desiredMonCount int, name string) bool {
|
||||
// make sure the failed mon is marked out of quorum
|
||||
if _, err := c.trackMonInOrOutOfQuorum(name, false); err != nil {
|
||||
logger.Errorf("failed to track failed mon %q. %v", name, err)
|
||||
}
|
||||
|
||||
if monCount > desiredMonCount {
|
||||
// no need to create a new mon since we have an extra
|
||||
if err := c.removeMon(name); err != nil {
|
||||
|
||||
@@ -19,6 +19,7 @@ package mon
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -28,6 +29,7 @@ import (
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/version"
|
||||
testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test"
|
||||
@@ -206,6 +208,51 @@ func TestRemoveExtraMon(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackMonsOutOfQuorum(t *testing.T) {
|
||||
endpoint := "1.2.3.4:6789"
|
||||
clientset := test.New(t, 1)
|
||||
tempDir, err := os.MkdirTemp("", "")
|
||||
assert.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
|
||||
c := &Cluster{
|
||||
mapping: &opcontroller.Mapping{},
|
||||
context: &clusterd.Context{Clientset: clientset, ConfigDir: tempDir},
|
||||
ownerInfo: ownerInfo,
|
||||
Namespace: "ns"}
|
||||
c.ClusterInfo = &cephclient.ClusterInfo{Monitors: map[string]*cephclient.MonInfo{
|
||||
"a": {Name: "a", Endpoint: endpoint},
|
||||
"b": {Name: "b", Endpoint: endpoint},
|
||||
"c": {Name: "c", Endpoint: endpoint},
|
||||
}}
|
||||
// No change since all mons are in quorum
|
||||
updated, err := c.trackMonInOrOutOfQuorum("a", true)
|
||||
assert.False(t, updated)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// initialize the configmap
|
||||
err = c.persistExpectedMonDaemons()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Track mon.a as out of quorum
|
||||
updated, err = c.trackMonInOrOutOfQuorum("a", false)
|
||||
assert.True(t, updated)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cm, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "a", cm.Data[controller.OutOfQuorumKey])
|
||||
|
||||
// Put mon.a back in quorum
|
||||
updated, err = c.trackMonInOrOutOfQuorum("a", true)
|
||||
assert.True(t, updated)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cm, err = clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "", cm.Data[controller.OutOfQuorumKey])
|
||||
}
|
||||
|
||||
func TestSkipMonFailover(t *testing.T) {
|
||||
c := New(context.TODO(), &clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil)
|
||||
c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
|
||||
|
||||
@@ -1097,6 +1097,14 @@ func (c *Cluster) persistExpectedMonDaemons() error {
|
||||
return errors.Wrap(err, "failed to save maxMonID")
|
||||
}
|
||||
|
||||
// preserve the mons detected out of quorum
|
||||
var monsOutOfQuorum []string
|
||||
for monName, mon := range c.ClusterInfo.Monitors {
|
||||
if mon.OutOfQuorum {
|
||||
monsOutOfQuorum = append(monsOutOfQuorum, monName)
|
||||
}
|
||||
}
|
||||
|
||||
configMap.Data = map[string]string{
|
||||
EndpointDataKey: FlattenMonEndpoints(c.ClusterInfo.Monitors),
|
||||
// persist the maxMonID that was previously stored in the configmap. We are likely saving info
|
||||
@@ -1104,9 +1112,10 @@ func (c *Cluster) persistExpectedMonDaemons() error {
|
||||
// actually been started. If the operator is restarted or the reconcile is otherwise restarted,
|
||||
// we want to calculate the mon scheduling next time based on the committed maxMonID, rather
|
||||
// than only a mon scheduling, which may not have completed.
|
||||
controller.MaxMonIDKey: maxMonID,
|
||||
controller.MappingKey: string(monMapping),
|
||||
csi.ConfigKey: csiConfigValue,
|
||||
controller.MaxMonIDKey: maxMonID,
|
||||
controller.MappingKey: string(monMapping),
|
||||
controller.OutOfQuorumKey: strings.Join(monsOutOfQuorum, ","),
|
||||
csi.ConfigKey: csiConfigValue,
|
||||
}
|
||||
|
||||
if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(c.ClusterInfo.Context, configMap, metav1.CreateOptions{}); err != nil {
|
||||
|
||||
@@ -56,6 +56,8 @@ const (
|
||||
EndpointConfigMapName = "rook-ceph-mon-endpoints"
|
||||
// EndpointDataKey is the name of the key inside the mon configmap to get the endpoints
|
||||
EndpointDataKey = "data"
|
||||
// OutOfQuorumKey is the name of the key for tracking mons detected out of quorum
|
||||
OutOfQuorumKey = "outOfQuorum"
|
||||
// MaxMonIDKey is the name of the max mon id used
|
||||
MaxMonIDKey = "maxMonId"
|
||||
// MappingKey is the name of the mapping for the mon->node and node->port
|
||||
@@ -249,6 +251,21 @@ func ExtractKey(contents string) (string, error) {
|
||||
return secret, nil
|
||||
}
|
||||
|
||||
func UpdateMonsOutOfQuorum(clientset kubernetes.Interface, namespace string, monsOutOfQuorum []string) error {
|
||||
ctx := context.TODO()
|
||||
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get mon endpoints configmap")
|
||||
}
|
||||
|
||||
cm.Data[OutOfQuorumKey] = strings.Join(monsOutOfQuorum, ",")
|
||||
_, err = clientset.CoreV1().ConfigMaps(namespace).Update(ctx, cm, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update mon endpoints configmap with mon(s) out of quorum")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadMonConfig returns the monitor endpoints and maxMonID
|
||||
func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string]*cephclient.MonInfo, int, *Mapping, error) {
|
||||
ctx := context.TODO()
|
||||
@@ -272,6 +289,18 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string
|
||||
monEndpointMap = ParseMonEndpoints(info)
|
||||
}
|
||||
|
||||
// Parse the mons that were detected out of quorum
|
||||
if outOfQuorum, ok := cm.Data[OutOfQuorumKey]; ok && len(outOfQuorum) > 0 {
|
||||
monNames := strings.Split(outOfQuorum, ",")
|
||||
for _, monName := range monNames {
|
||||
if monInfo, ok := monEndpointMap[monName]; ok {
|
||||
monInfo.OutOfQuorum = true
|
||||
} else {
|
||||
logger.Warningf("did not find mon %q to set it out of quorum in the cluster info", monName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse the max monitor id
|
||||
storedMaxMonID := -1
|
||||
if id, ok := cm.Data[MaxMonIDKey]; ok {
|
||||
|
||||
Reference in new issue
Block a user