Files
my-rook-config/pkg/cephmgr/mon/leader.go
T

454 lines
13 KiB
Go

/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mon
import (
"errors"
"fmt"
"path"
"strconv"
"strings"
"time"
ctx "golang.org/x/net/context"
etcd "github.com/coreos/etcd/client"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
"github.com/rook/rook/pkg/util"
)
const (
monitorKey = "monitor"
UnhealthyHeartbeatAgeSeconds = 10
)
type Leader struct {
waitForQuorum func(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error
}
func NewLeader() *Leader {
return &Leader{waitForQuorum: WaitForQuorum}
}
// Apply the desired state to the cluster. The context provides all the information needed to make changes to the service.
// Create the ceph monitors
// Must be idempotent
func (m *Leader) Configure(context *clusterd.Context, factory client.ConnectionFactory, adminSecret string) error {
// Create or get the basic cluster info
var err error
cluster, err := createOrGetClusterInfo(factory, context.EtcdClient, adminSecret)
if err != nil {
return err
}
logger.Infof("Creating monitors with %d nodes available", len(context.Inventory.Nodes))
// choose the nodes where the monitors will run
var monitorsToRemove map[string]*CephMonitorConfig
cluster.Monitors, monitorsToRemove, err = chooseMonitorNodes(context)
if err != nil {
logger.Errorf("failed to choose monitors. err=%s", err.Error())
return err
}
// trigger the monitors to start on each node
err = clusterd.TriggerAgentsAndWaitForCompletion(context.EtcdClient, monIDs(cluster.Monitors), monitorAgentName, len(cluster.Monitors))
if err != nil {
return err
}
// wait for quorum
err = m.waitForQuorum(factory, context, cluster)
if err != nil {
return err
}
if len(monitorsToRemove) == 0 {
// no monitors to remove
return nil
}
// notify the quorum to remove the bad members
err = removeMonitorsFromQuorum(factory, context, cluster, monitorsToRemove)
if err != nil {
logger.Warningf("failed to remove monitors from quorum. %v", err)
}
return nil
}
func removeMonitorsFromQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo, monitors map[string]*CephMonitorConfig) error {
// trigger the monitors to remove, but don't wait for a response very long since they are likely down
waitSeconds := 10
err := clusterd.TriggerAgentsAndWait(context.EtcdClient, monIDs(monitors), monitorAgentName, 0, waitSeconds)
if err != nil {
return fmt.Errorf("failed to trigger removal of unhealthy monitors. %v", err)
}
// open an admin connection to the cluster
conn, err := ConnectToClusterAsAdmin(context, factory, cluster)
if err != nil {
return err
}
defer conn.Shutdown()
logger.Infof("removing %d monitors from quorum", len(monitors))
for _, mon := range monitors {
if err := RemoveMonitorFromQuorum(conn, mon.Name); err != nil {
return err
}
}
return nil
}
func RemoveMonitorFromQuorum(conn client.Connection, name string) error {
logger.Debugf("removing monitor %s", name)
cmd := map[string]interface{}{"prefix": "mon remove", "name": name}
_, err := client.ExecuteMonCommand(conn, cmd, "mon remove")
if err != nil {
return fmt.Errorf("mon %s remove failed: %+v", name, err)
}
logger.Infof("removed monitor %s", name)
return nil
}
// extract the nodeIDs from the mon map
func monIDs(mons map[string]*CephMonitorConfig) []string {
nodes := []string{}
for mon := range mons {
nodes = append(nodes, mon)
}
return nodes
}
func monsOnUnhealthyNode(context *clusterd.Context, nodes []*clusterd.UnhealthyNode) (bool, error) {
cluster, err := LoadClusterInfo(context.EtcdClient)
if err != nil {
return false, fmt.Errorf("failed to load cluster info: %+v", err)
}
for _, node := range nodes {
if isMonitor(cluster, node.ID) {
return true, nil
}
}
return false, nil
}
func isMonitor(cluster *ClusterInfo, nodeID string) bool {
for mon := range cluster.Monitors {
if mon == nodeID {
return true
}
}
return false
}
func GetDesiredMonitors(etcdClient etcd.KeysAPI) (map[string]*CephMonitorConfig, error) {
// query the desired monitors from etcd
monitors := make(map[string]*CephMonitorConfig)
monKey := path.Join(CephKey, monitorKey, clusterd.DesiredKey)
previousMonitors, err := etcdClient.Get(ctx.Background(), monKey, &etcd.GetOptions{Recursive: true})
if err != nil {
if util.IsEtcdKeyNotFound(err) {
return monitors, nil
}
return nil, err
}
if previousMonitors == nil || previousMonitors.Node == nil {
return monitors, nil
}
// parse the monitor info from etcd
for _, node := range previousMonitors.Node.Nodes {
nodeID := util.GetLeafKeyPath(node.Key)
mon := &CephMonitorConfig{}
ipaddress := ""
port := ""
for _, monSettings := range node.Nodes {
if strings.HasSuffix(monSettings.Key, "/id") {
mon.Name = monSettings.Value
} else if strings.HasSuffix(monSettings.Key, "/ipaddress") {
ipaddress = monSettings.Value
} else if strings.HasSuffix(monSettings.Key, "/port") {
port = monSettings.Value
}
}
if mon.Name == "" || ipaddress == "" || port == "" {
return nil, errors.New("missing monitor id or ip address or port")
}
mon.Endpoint = fmt.Sprintf("%s:%s", ipaddress, port)
monitors[nodeID] = mon
}
return monitors, nil
}
func chooseMonitorNodes(context *clusterd.Context) (map[string]*CephMonitorConfig, map[string]*CephMonitorConfig, error) {
nodeCount := len(context.Inventory.Nodes)
// calculate how many monitors are desired
desiredMonitors := calculateMonitorCount(nodeCount)
// get the monitors that have already been chosen
monitors, err := GetDesiredMonitors(context.EtcdClient)
if err != nil {
return nil, nil, err
}
// get the unhealthy monitors
monitorsToRemove := getUnhealthyMonitors(context, monitors)
newMons := desiredMonitors + len(monitorsToRemove) - len(monitors)
logger.Infof("Monitor state. current=%d, desired=%d, unhealthy=%d, toAdd=%d", len(monitors), desiredMonitors, len(monitorsToRemove), newMons)
if newMons <= 0 {
logger.Debugf("No need for new monitors")
return monitors, monitorsToRemove, nil
}
// Select nodes and assign them a monitor ID
nextMonID, err := getMaxMonitorID(monitors)
if err != nil {
return nil, nil, fmt.Errorf("failed to get max monitor id. %v", err)
}
// increment the id because we were actually given the max known id above and we need the next desired id
nextMonID++
// iterate through the monitors to find the new candidates
var settings = make(map[string]string)
addedMons := 0
for nodeID := range context.Inventory.Nodes {
// skip the node if already in the list of monitors
if mon, ok := monitors[nodeID]; ok {
logger.Debugf("skipping node %s that is %s", nodeID, mon.Name)
continue
}
node, ok := context.Inventory.Nodes[nodeID]
if !ok || node.PublicIP == "" {
logger.Errorf("failed to discover desired ip address for node %s. %v", nodeID, err)
return nil, nil, err
}
if !isNodeHealthyForMon(node) {
logger.Infof("skipping selection of unhealthy node %s as a monitor (age=%s)", nodeID, node.HeartbeatAge)
continue
}
// Store the monitor id and connection info
port := "6790"
monitorID := fmt.Sprintf("mon%d", nextMonID)
settings[path.Join(nodeID, "id")] = monitorID
settings[path.Join(nodeID, "ipaddress")] = node.PublicIP
settings[path.Join(nodeID, "port")] = port
monitor := &CephMonitorConfig{Name: monitorID, Endpoint: fmt.Sprintf("%s:%s", node.PublicIP, port)}
monitors[nodeID] = monitor
nextMonID++
addedMons++
// break if we have enough monitors now
if addedMons == newMons {
break
}
}
if addedMons != newMons {
return monitors, monitorsToRemove, fmt.Errorf("only added %d/%d expected new monitors. aborting monitor deployment.", addedMons, newMons)
}
// FIX: Use an etcd3 transaction to add the monitor keys and remove unhealhty monitors
// store the properties for the new monitors
monKey := path.Join(CephKey, monitorKey, clusterd.DesiredKey)
err = util.StoreEtcdProperties(context.EtcdClient, monKey, settings)
if err != nil {
logger.Errorf("failed to save monitor ids. err=%v", err)
return nil, nil, err
}
// remove the unhealthy monitors from the desired state
for monID, monitor := range monitorsToRemove {
logger.Infof("removing monitor %s (%+v) from desired state", monID, monitor)
key := path.Join(monKey, monID)
_, err = context.EtcdClient.Delete(ctx.Background(), key, &etcd.DeleteOptions{Dir: true, Recursive: true})
if err != nil {
return nil, nil, fmt.Errorf("failed to remove monitor %s from desired state. %v", monID, err)
}
delete(monitors, monID)
}
return monitors, monitorsToRemove, nil
}
func isNodeHealthyForMon(node *inventory.NodeConfig) bool {
return int(node.HeartbeatAge.Seconds()) < UnhealthyHeartbeatAgeSeconds
}
func getUnhealthyMonitors(context *clusterd.Context, monitors map[string]*CephMonitorConfig) map[string]*CephMonitorConfig {
unhealthyMons := make(map[string]*CephMonitorConfig)
for monID, mon := range monitors {
node, ok := context.Inventory.Nodes[monID]
// if the monitor is not found in the inventory or else it has an unhealthy heartbeat, add it to the list
if !ok || !isNodeHealthyForMon(node) {
unhealthyMons[monID] = mon
}
}
return unhealthyMons
}
// get the highest monitor ID from the list of montors
func getMaxMonitorID(monitors map[string]*CephMonitorConfig) (int, error) {
maxMonitorID := -1
for _, mon := range monitors {
// monitors are expected to have a name of "mon" with an integer suffix
if len(mon.Name) < 4 || mon.Name[0:3] != "mon" {
return 0, fmt.Errorf("invalid monitor id %s", mon.Name)
}
substr := mon.Name[3:]
id, err := strconv.ParseInt(substr, 10, 32)
if err != nil {
return 0, fmt.Errorf("bad monitor id %s. %v", mon.Name, err)
}
if int(id) > maxMonitorID {
maxMonitorID = int(id)
}
}
return maxMonitorID, nil
}
// Calculate the number of monitors that should be deployed
func calculateMonitorCount(nodeCount int) int {
switch {
case nodeCount > 100:
return 7
case nodeCount > 20:
return 5
case nodeCount > 2:
return 3
case nodeCount > 0:
return 1
default:
return 0
}
}
func WaitForQuorum(factory client.ConnectionFactory, context *clusterd.Context, cluster *ClusterInfo) error {
// open an admin connection to the cluster
adminConn, err := ConnectToClusterAsAdmin(context, factory, cluster)
if err != nil {
return err
}
defer adminConn.Shutdown()
mons := []string{}
for _, m := range cluster.Monitors {
mons = append(mons, m.Name)
}
return WaitForQuorumWithConnection(adminConn, mons)
}
// Wait for all of the given monitors to form quorum
func WaitForQuorumWithConnection(conn client.Connection, mons []string) error {
logger.Infof("waiting for mon quorum")
// wait for monitors to establish quorum
retryCount := 0
retryMax := 20
sleepTime := 5
for {
retryCount++
if retryCount > retryMax {
return fmt.Errorf("exceeded max retry count waiting for monitors to reach quorum")
}
if retryCount > 1 {
// only sleep after the first time
<-time.After(time.Duration(sleepTime) * time.Second)
}
// get the mon_status response that contains info about all monitors in the mon map and
// their quorum status
monStatusResp, err := client.GetMonStatus(conn)
if err != nil {
logger.Debugf("failed to get mon_status, err: %+v", err)
continue
}
// check if each of the initial monitors is in quorum
allInQuorum := true
for _, name := range mons {
// first get the initial monitors corresponding mon map entry
var monMapEntry *client.MonMapEntry
for i := range monStatusResp.MonMap.Mons {
if name == monStatusResp.MonMap.Mons[i].Name {
monMapEntry = &monStatusResp.MonMap.Mons[i]
break
}
}
if monMapEntry == nil {
// found an initial monitor that is not in the mon map, bail out of this retry
logger.Warningf("failed to find initial monitor %s in mon map", name)
allInQuorum = false
break
}
// using the current initial monitor's mon map entry, check to see if it's in the quorum list
// (a list of monitor rank values)
inQuorumList := false
for _, q := range monStatusResp.Quorum {
if monMapEntry.Rank == q {
inQuorumList = true
break
}
}
if !inQuorumList {
// found an initial monitor that is not in quorum, bail out of this retry
logger.Warningf("initial monitor %s is not in quorum list", name)
allInQuorum = false
break
}
}
if allInQuorum {
logger.Debugf("all initial monitors are in quorum")
break
}
}
logger.Infof("Ceph monitors formed quorum")
return nil
}