From aa9f4088227da6557f18de68892f60672cfa2a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 24 Oct 2019 11:27:52 +0200 Subject: [PATCH] ceph: use quorum_status instead of mon_status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Octopus, this commit https://github.com/ceph/ceph/pull/30859/commits/c09b82a80a392ccd0da7677c7b424ce5cd3fa5d6 moved the quorum_status command to a socket call. Thus, the `ceph quorum_status` command does not exist anymore, to use it we need to do `ceph daemon mon.$ID quorum_status`. However, this command only works locally so it won't work if we run it from the operator. The `ceph mon_status` command is a general command that run from anywhere and returns the mon map, let's use this one on the operator code. This is not a breaking change. Signed-off-by: Sébastien Han --- pkg/daemon/ceph/client/command_test.go | 8 +++---- pkg/daemon/ceph/client/mon.go | 10 ++++---- pkg/operator/ceph/cluster/mon/health.go | 32 ++++++++++++------------- pkg/operator/ceph/cluster/mon/mon.go | 28 +++++++++++----------- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/pkg/daemon/ceph/client/command_test.go b/pkg/daemon/ceph/client/command_test.go index 122bf0c98..c3b3eff1d 100644 --- a/pkg/daemon/ceph/client/command_test.go +++ b/pkg/daemon/ceph/client/command_test.go @@ -27,9 +27,9 @@ func TestFinalizeCephCommandArgs(t *testing.T) { clusterName := "rook" configDir := "/var/lib/rook/rook-ceph" expectedCommand := "ceph" - args := []string{"mon_status"} + args := []string{"quorum_status"} expectedArgs := []string{ - "mon_status", + "quorum_status", "--connect-timeout=15", "--cluster=rook", "--conf=/var/lib/rook/rook-ceph/rook/rook.config", @@ -98,9 +98,9 @@ func TestFinalizeCephCommandArgsClusterDefaultName(t *testing.T) { clusterName := "ceph" configDir := "/etc" expectedCommand := "ceph" - args := []string{"mon_status"} + args := []string{"quorum_status"} expectedArgs := []string{ - "mon_status", + "quorum_status", "--connect-timeout=15", } diff --git a/pkg/daemon/ceph/client/mon.go b/pkg/daemon/ceph/client/mon.go index 7b812bda9..6a22c8d65 100644 --- a/pkg/daemon/ceph/client/mon.go +++ b/pkg/daemon/ceph/client/mon.go @@ -22,7 +22,7 @@ import ( "github.com/rook/rook/pkg/clusterd" ) -// represents the response from a mon_status mon_command (subset of all available fields, only +// MonStatusResponse represents the response from a quorum_status mon_command (subset of all available fields, only // marshal ones we care about) type MonStatusResponse struct { Quorum []int `json:"quorum"` @@ -60,14 +60,14 @@ type AddrvecEntry struct { Nonce int `json:"nonce"` } -// GetMonStatus calls mon_status mon_command -func GetMonStatus(context *clusterd.Context, clusterName string, debug bool) (MonStatusResponse, error) { - args := []string{"mon_status"} +// GetMonQuorumStatus calls quorum_status mon_command +func GetMonQuorumStatus(context *clusterd.Context, clusterName string, debug bool) (MonStatusResponse, error) { + args := []string{"quorum_status"} cmd := NewCephCommand(context, clusterName, args) cmd.Debug = debug buf, err := cmd.Run() if err != nil { - return MonStatusResponse{}, fmt.Errorf("mon status failed. %+v", err) + return MonStatusResponse{}, fmt.Errorf("mon quorum status failed. %+v", err) } var resp MonStatusResponse diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 052f47161..4d0766193 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -83,13 +83,13 @@ func (c *Cluster) checkHealth() error { // connect to the mons // get the status and check for quorum - status, err := client.GetMonStatus(c.context, c.ClusterInfo.Name, true) + quorumStatus, err := client.GetMonQuorumStatus(c.context, c.ClusterInfo.Name, true) if err != nil { - return fmt.Errorf("failed to get mon status. %+v", err) + return fmt.Errorf("failed to get mon quorum status. %+v", err) } - logger.Debugf("Mon status: %+v", status) + logger.Debugf("Mon quorum status: %+v", quorumStatus) if c.spec.External.Enable { - return c.handleExternalMonStatus(status) + return c.handleExternalMonStatus(quorumStatus) } // Use a local mon count in case the user updates the crd in another goroutine. @@ -106,8 +106,8 @@ func (c *Cluster) checkHealth() error { // first handle mons that are not in quorum but in the ceph mon map // failover the unhealthy mons allMonsInQuorum := true - for _, mon := range status.MonMap.Mons { - inQuorum := monInQuorum(mon, status.Quorum) + for _, mon := range quorumStatus.MonMap.Mons { + inQuorum := monInQuorum(mon, quorumStatus.Quorum) // if the mon is in quorum remove it from our check for "existence" // else see below condition if _, ok := monsNotFound[mon.Name]; ok { @@ -115,7 +115,7 @@ func (c *Cluster) checkHealth() error { } else { // when the mon isn't in the clusterInfo, but is in quorum and there are // enough mons, remove it else remove it on the next run - if inQuorum && len(status.MonMap.Mons) > desiredMonCount { + if inQuorum && len(quorumStatus.MonMap.Mons) > desiredMonCount { logger.Warningf("mon %s not in source of truth but in quorum, removing", mon.Name) c.removeMon(mon.Name) } else { @@ -123,7 +123,7 @@ func (c *Cluster) checkHealth() error { "mon %s not in source of truth and not in quorum, not enough mons to remove now (wanted: %d, current: %d)", mon.Name, desiredMonCount, - len(status.MonMap.Mons), + len(quorumStatus.MonMap.Mons), ) } } @@ -136,7 +136,7 @@ func (c *Cluster) checkHealth() error { logger.Infof("mon %s is back in quorum, removed from mon out timeout list", mon.Name) } } else { - logger.Debugf("mon %s NOT found in quorum. Mon status: %+v", mon.Name, status) + logger.Debugf("mon %s NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus) allMonsInQuorum = false // If not yet set, add the current time, for the timeout @@ -153,7 +153,7 @@ func (c *Cluster) checkHealth() error { } logger.Warningf("mon %s NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name) - c.failMon(len(status.MonMap.Mons), desiredMonCount, mon.Name) + c.failMon(len(quorumStatus.MonMap.Mons), desiredMonCount, mon.Name) // only deal with one unhealthy mon per health check return nil } @@ -169,18 +169,18 @@ func (c *Cluster) checkHealth() error { } // create/start new mons when there are fewer mons than the desired count in the CRD - if len(status.MonMap.Mons) < desiredMonCount { - logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(status.MonMap.Mons), desiredMonCount) + if len(quorumStatus.MonMap.Mons) < desiredMonCount { + logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(quorumStatus.MonMap.Mons), desiredMonCount) return c.startMons(desiredMonCount) } // remove extra mons if the desired count has decreased in the CRD and all the mons are currently healthy - if allMonsInQuorum && len(status.MonMap.Mons) > desiredMonCount { - if desiredMonCount < 2 && len(status.MonMap.Mons) == 2 { + if allMonsInQuorum && len(quorumStatus.MonMap.Mons) > desiredMonCount { + if desiredMonCount < 2 && len(quorumStatus.MonMap.Mons) == 2 { logger.Warningf("cannot reduce mon quorum size from 2 to 1") } else { - logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(status.MonMap.Mons), desiredMonCount) - return c.removeMon(status.MonMap.Mons[0].Name) + logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount) + return c.removeMon(quorumStatus.MonMap.Mons[0].Name) } } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 22a2bd2bf..ef4cb3be4 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -943,23 +943,23 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [ continue } - // get the mon_status response that contains info about all monitors in the mon map and + // get the quorum_status response that contains info about all monitors in the mon map and // their quorum status - monStatusResp, err := client.GetMonStatus(context, clusterName, false) + monQuorumStatusResp, err := client.GetMonQuorumStatus(context, clusterName, false) if err != nil { - logger.Debugf("failed to get mon_status, err: %+v", err) + logger.Debugf("failed to get quorum_status. %+v", err) continue } if !requireAllInQuorum { - logQuorumMembers(monStatusResp) + logQuorumMembers(monQuorumStatusResp) break } // check if each of the initial monitors is in quorum allInQuorum := true for _, name := range mons { - if !monFoundInQuorum(name, monStatusResp) { + if !monFoundInQuorum(name, monQuorumStatusResp) { // found an initial monitor that is not in quorum, bail out of this retry logger.Warningf("monitor %s is not in quorum list", name) allInQuorum = false @@ -968,7 +968,7 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [ } if allInQuorum { - logQuorumMembers(monStatusResp) + logQuorumMembers(monQuorumStatusResp) break } } @@ -976,22 +976,22 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [ return nil } -func logQuorumMembers(monStatusResp client.MonStatusResponse) { +func logQuorumMembers(monQuorumStatusResp client.MonStatusResponse) { var monsInQuorum []string - for _, m := range monStatusResp.MonMap.Mons { - if monFoundInQuorum(m.Name, monStatusResp) { + for _, m := range monQuorumStatusResp.MonMap.Mons { + if monFoundInQuorum(m.Name, monQuorumStatusResp) { monsInQuorum = append(monsInQuorum, m.Name) } } logger.Infof("Monitors in quorum: %v", monsInQuorum) } -func monFoundInQuorum(name string, monStatusResp client.MonStatusResponse) bool { +func monFoundInQuorum(name string, monQuorumStatusResp client.MonStatusResponse) bool { // first get the initial monitors corresponding mon map entry var monMapEntry *client.MonMapEntry - for i := range monStatusResp.MonMap.Mons { - if name == monStatusResp.MonMap.Mons[i].Name { - monMapEntry = &monStatusResp.MonMap.Mons[i] + for i := range monQuorumStatusResp.MonMap.Mons { + if name == monQuorumStatusResp.MonMap.Mons[i].Name { + monMapEntry = &monQuorumStatusResp.MonMap.Mons[i] break } } @@ -1004,7 +1004,7 @@ func monFoundInQuorum(name string, monStatusResp client.MonStatusResponse) bool // using the current initial monitor's mon map entry, check to see if it's in the quorum list // (a list of monitor rank values) - for _, q := range monStatusResp.Quorum { + for _, q := range monQuorumStatusResp.Quorum { if monMapEntry.Rank == q { return true }