feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #4168 from leseb/use-quorum_status

Browse Source 
ceph: use quorum_status instead of mon_status
This commit is contained in:
Travis Nielsen
2019-10-24 16:14:42 -06:00
committed by GitHub
parent 8b8448b543 aa9f408822
commit 092b046f2c
4 changed files with 39 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/daemon/ceph/client/command_test.go
+4 -4
View File
@@ -27,9 +27,9 @@ func TestFinalizeCephCommandArgs(t *testing.T) {
clusterName := "rook"
configDir := "/var/lib/rook/rook-ceph"
expectedCommand := "ceph"
args := []string{"mon_status"}
args := []string{"quorum_status"}
expectedArgs := []string{
"mon_status",
"quorum_status",
"--connect-timeout=15",
"--cluster=rook",
"--conf=/var/lib/rook/rook-ceph/rook/rook.config",
@@ -98,9 +98,9 @@ func TestFinalizeCephCommandArgsClusterDefaultName(t *testing.T) {
clusterName := "ceph"
configDir := "/etc"
expectedCommand := "ceph"
args := []string{"mon_status"}
args := []string{"quorum_status"}
expectedArgs := []string{
"mon_status",
"quorum_status",
"--connect-timeout=15",
}
pkg/daemon/ceph/client/mon.go
+5 -5
View File
@@ -22,7 +22,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
)
// represents the response from a mon_status mon_command (subset of all available fields, only
// MonStatusResponse represents the response from a quorum_status mon_command (subset of all available fields, only
// marshal ones we care about)
type MonStatusResponse struct {
Quorum []int `json:"quorum"`
@@ -60,14 +60,14 @@ type AddrvecEntry struct {
Nonce int `json:"nonce"`
}
// GetMonStatus calls mon_status mon_command
func GetMonStatus(context *clusterd.Context, clusterName string, debug bool) (MonStatusResponse, error) {
args := []string{"mon_status"}
// GetMonQuorumStatus calls quorum_status mon_command
func GetMonQuorumStatus(context *clusterd.Context, clusterName string, debug bool) (MonStatusResponse, error) {
args := []string{"quorum_status"}
cmd := NewCephCommand(context, clusterName, args)
cmd.Debug = debug
buf, err := cmd.Run()
if err != nil {
return MonStatusResponse{}, fmt.Errorf("mon status failed. %+v", err)
return MonStatusResponse{}, fmt.Errorf("mon quorum status failed. %+v", err)
}
var resp MonStatusResponse
pkg/operator/ceph/cluster/mon/health.go
+16 -16
View File
@@ -83,13 +83,13 @@ func (c *Cluster) checkHealth() error {
// connect to the mons
// get the status and check for quorum
status, err := client.GetMonStatus(c.context, c.ClusterInfo.Name, true)
quorumStatus, err := client.GetMonQuorumStatus(c.context, c.ClusterInfo.Name, true)
if err != nil {
return fmt.Errorf("failed to get mon status. %+v", err)
return fmt.Errorf("failed to get mon quorum status. %+v", err)
}
logger.Debugf("Mon status: %+v", status)
logger.Debugf("Mon quorum status: %+v", quorumStatus)
if c.spec.External.Enable {
return c.handleExternalMonStatus(status)
return c.handleExternalMonStatus(quorumStatus)
}
// Use a local mon count in case the user updates the crd in another goroutine.
@@ -106,8 +106,8 @@ func (c *Cluster) checkHealth() error {
// first handle mons that are not in quorum but in the ceph mon map
// failover the unhealthy mons
allMonsInQuorum := true
for _, mon := range status.MonMap.Mons {
inQuorum := monInQuorum(mon, status.Quorum)
for _, mon := range quorumStatus.MonMap.Mons {
inQuorum := monInQuorum(mon, quorumStatus.Quorum)
// if the mon is in quorum remove it from our check for "existence"
// else see below condition
if _, ok := monsNotFound[mon.Name]; ok {
@@ -115,7 +115,7 @@ func (c *Cluster) checkHealth() error {
} else {
// when the mon isn't in the clusterInfo, but is in quorum and there are
// enough mons, remove it else remove it on the next run
if inQuorum && len(status.MonMap.Mons) > desiredMonCount {
if inQuorum && len(quorumStatus.MonMap.Mons) > desiredMonCount {
logger.Warningf("mon %s not in source of truth but in quorum, removing", mon.Name)
c.removeMon(mon.Name)
} else {
@@ -123,7 +123,7 @@ func (c *Cluster) checkHealth() error {
"mon %s not in source of truth and not in quorum, not enough mons to remove now (wanted: %d, current: %d)",
mon.Name,
desiredMonCount,
len(status.MonMap.Mons),
len(quorumStatus.MonMap.Mons),
)
}
}
@@ -136,7 +136,7 @@ func (c *Cluster) checkHealth() error {
logger.Infof("mon %s is back in quorum, removed from mon out timeout list", mon.Name)
}
} else {
logger.Debugf("mon %s NOT found in quorum. Mon status: %+v", mon.Name, status)
logger.Debugf("mon %s NOT found in quorum. Mon quorum status: %+v", mon.Name, quorumStatus)
allMonsInQuorum = false
// If not yet set, add the current time, for the timeout
@@ -153,7 +153,7 @@ func (c *Cluster) checkHealth() error {
}
logger.Warningf("mon %s NOT found in quorum and timeout exceeded, mon will be failed over", mon.Name)
c.failMon(len(status.MonMap.Mons), desiredMonCount, mon.Name)
c.failMon(len(quorumStatus.MonMap.Mons), desiredMonCount, mon.Name)
// only deal with one unhealthy mon per health check
return nil
}
@@ -169,18 +169,18 @@ func (c *Cluster) checkHealth() error {
}
// create/start new mons when there are fewer mons than the desired count in the CRD
if len(status.MonMap.Mons) < desiredMonCount {
logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(status.MonMap.Mons), desiredMonCount)
if len(quorumStatus.MonMap.Mons) < desiredMonCount {
logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(quorumStatus.MonMap.Mons), desiredMonCount)
return c.startMons(desiredMonCount)
}
// remove extra mons if the desired count has decreased in the CRD and all the mons are currently healthy
if allMonsInQuorum && len(status.MonMap.Mons) > desiredMonCount {
if desiredMonCount < 2 && len(status.MonMap.Mons) == 2 {
if allMonsInQuorum && len(quorumStatus.MonMap.Mons) > desiredMonCount {
if desiredMonCount < 2 && len(quorumStatus.MonMap.Mons) == 2 {
logger.Warningf("cannot reduce mon quorum size from 2 to 1")
} else {
logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(status.MonMap.Mons), desiredMonCount)
return c.removeMon(status.MonMap.Mons[0].Name)
logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount)
return c.removeMon(quorumStatus.MonMap.Mons[0].Name)
}
}
pkg/operator/ceph/cluster/mon/mon.go
+14 -14
View File
@@ -943,23 +943,23 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [
continue
}
// get the mon_status response that contains info about all monitors in the mon map and
// get the quorum_status response that contains info about all monitors in the mon map and
// their quorum status
monStatusResp, err := client.GetMonStatus(context, clusterName, false)
monQuorumStatusResp, err := client.GetMonQuorumStatus(context, clusterName, false)
if err != nil {
logger.Debugf("failed to get mon_status, err: %+v", err)
logger.Debugf("failed to get quorum_status. %+v", err)
continue
}
if !requireAllInQuorum {
logQuorumMembers(monStatusResp)
logQuorumMembers(monQuorumStatusResp)
break
}
// check if each of the initial monitors is in quorum
allInQuorum := true
for _, name := range mons {
if !monFoundInQuorum(name, monStatusResp) {
if !monFoundInQuorum(name, monQuorumStatusResp) {
// found an initial monitor that is not in quorum, bail out of this retry
logger.Warningf("monitor %s is not in quorum list", name)
allInQuorum = false
@@ -968,7 +968,7 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [
}
if allInQuorum {
logQuorumMembers(monStatusResp)
logQuorumMembers(monQuorumStatusResp)
break
}
}
@@ -976,22 +976,22 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [
return nil
}
func logQuorumMembers(monStatusResp client.MonStatusResponse) {
func logQuorumMembers(monQuorumStatusResp client.MonStatusResponse) {
var monsInQuorum []string
for _, m := range monStatusResp.MonMap.Mons {
if monFoundInQuorum(m.Name, monStatusResp) {
for _, m := range monQuorumStatusResp.MonMap.Mons {
if monFoundInQuorum(m.Name, monQuorumStatusResp) {
monsInQuorum = append(monsInQuorum, m.Name)
}
}
logger.Infof("Monitors in quorum: %v", monsInQuorum)
}
func monFoundInQuorum(name string, monStatusResp client.MonStatusResponse) bool {
func monFoundInQuorum(name string, monQuorumStatusResp client.MonStatusResponse) bool {
// first get the initial monitors corresponding mon map entry
var monMapEntry *client.MonMapEntry
for i := range monStatusResp.MonMap.Mons {
if name == monStatusResp.MonMap.Mons[i].Name {
monMapEntry = &monStatusResp.MonMap.Mons[i]
for i := range monQuorumStatusResp.MonMap.Mons {
if name == monQuorumStatusResp.MonMap.Mons[i].Name {
monMapEntry = &monQuorumStatusResp.MonMap.Mons[i]
break
}
}
@@ -1004,7 +1004,7 @@ func monFoundInQuorum(name string, monStatusResp client.MonStatusResponse) bool
// using the current initial monitor's mon map entry, check to see if it's in the quorum list
// (a list of monitor rank values)
for _, q := range monStatusResp.Quorum {
for _, q := range monQuorumStatusResp.Quorum {
if monMapEntry.Rank == q {
return true
}