my-rook-config/pkg/operator/ceph/cluster/mon/node.go

/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mon

import (
	"fmt"

	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/operator/k8sutil"
	"github.com/rook/rook/pkg/util"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// NodeUsage is a mapping between a Node and computed metadata about the node
// that is used in monitor pod scheduling.
type NodeUsage struct {
	Node *v1.Node
	// The number of monitor pods assigned to the node
	MonCount int
	// The node is available for scheduling monitor pods. This is equivalent to
	// evaluating k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
	MonValid bool
}

// getMonNodes detects the nodes that are available for new mons to start.
func (c *Cluster) getMonNodes() ([]v1.Node, error) {
	availableNodes, nodes, err := c.getAvailableMonNodes()
	if err != nil {
		return nil, err
	}
	logger.Infof("Found %d running nodes without mons", len(availableNodes))

	// if all nodes already have mons and the user has given the mon.count, add all nodes to be available
	if c.spec.Mon.AllowMultiplePerNode && len(availableNodes) == 0 {
		logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items))
		for _, node := range nodes.Items {
			valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
			if err != nil {
				logger.Warning("failed to validate node %s %v", node.Name, err)
			} else if valid {
				availableNodes = append(availableNodes, node)
			}
		}
	}

	return availableNodes, nil
}

func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) {
	nodeOptions := metav1.ListOptions{}
	nodeOptions.TypeMeta.Kind = "Node"
	nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
	if err != nil {
		return nil, nil, err
	}
	logger.Debugf("there are %d nodes available for %d mons", len(nodes.Items), len(c.ClusterInfo.Monitors))

	// get the nodes that have mons assigned
	nodesInUse, err := c.getNodesWithMons(nodes)
	if err != nil {
		logger.Warningf("could not get nodes with mons. %+v", err)
		nodesInUse = util.NewSet()
	}

	// choose nodes for the new mons that don't have mons currently
	availableNodes := []v1.Node{}
	for _, node := range nodes.Items {
		if !nodesInUse.Contains(node.Name) {
			valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
			if err != nil {
				logger.Warning("failed to validate node %s %v", node.Name, err)
			} else if valid {
				availableNodes = append(availableNodes, node)
			}
		}
	}

	return availableNodes, nodes, nil
}

func (c *Cluster) getNodesWithMons(nodes *v1.NodeList) (*util.Set, error) {
	// get the mon pods and their node affinity
	options := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}
	pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(options)
	if err != nil {
		return nil, err
	}
	nodesInUse := util.NewSet()
	for _, pod := range pods.Items {
		hostname := pod.Spec.NodeSelector[v1.LabelHostname]
		logger.Debugf("mon pod on node %s", hostname)
		name, ok := getNodeNameFromHostname(nodes, hostname)
		if !ok {
			logger.Errorf("mon %s on hostname %s not found in node list", pod.Name, hostname)
		}
		nodesInUse.Add(name)
	}
	return nodesInUse, nil
}

// Get the number of mons that the operator should be starting
func (c *Cluster) getTargetMonCount() (int, string, error) {

	// Get the full list of k8s nodes
	nodeOptions := metav1.ListOptions{}
	nodeOptions.TypeMeta.Kind = "Node"
	nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
	if err != nil {
		return 0, "", fmt.Errorf("failed to get nodes. %+v", err)
	}

	// Get the nodes where it is possible to place mons
	availableNodes := []v1.Node{}
	for _, node := range nodes.Items {
		valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
		if err != nil {
			logger.Warning("failed to validate node %s %v", node.Name, err)
		} else if valid {
			availableNodes = append(availableNodes, node)
		}
	}

	target, msg := calcTargetMonCount(len(availableNodes), c.spec.Mon)
	return target, msg, nil
}

func calcTargetMonCount(nodes int, spec cephv1.MonSpec) (int, string) {
	minTarget := spec.Count
	preferredTarget := spec.PreferredCount

	if preferredTarget <= minTarget {
		msg := fmt.Sprintf("targeting the mon count %d", minTarget)
		return minTarget, msg
	}
	if nodes >= preferredTarget {
		msg := fmt.Sprintf("targeting the preferred mon count %d since there are %d available nodes", preferredTarget, nodes)
		return preferredTarget, msg
	}
	if spec.AllowMultiplePerNode {
		msg := fmt.Sprintf("targeting the preferred mon count %d even if not that many nodes since multiple mons are allowed per node", preferredTarget)
		return preferredTarget, msg
	}
	if nodes <= minTarget {
		msg := fmt.Sprintf("targeting the min mon count %d since there are only %d available nodes", minTarget, nodes)
		return minTarget, msg
	}

	// There are between minTarget and preferredTarget nodes. Find an odd number for mons closest to the number of nodes.
	intermediate := nodes
	if intermediate%2 == 0 {
		// Decrease to an odd number if there are an even number of nodes
		intermediate--
	}
	msg := fmt.Sprintf("targeting the calculated mon count %d since there are %d available nodes", intermediate, nodes)
	return intermediate, msg
}

// Look up the immutable node name from the hostname label
func getNodeNameFromHostname(nodes *v1.NodeList, hostname string) (string, bool) {
	for _, node := range nodes.Items {
		if node.Labels[v1.LabelHostname] == hostname {
			return node.Name, true
		}
		if node.Name == hostname {
			return node.Name, true
		}
	}
	return "", false
}

func getNodeInfoFromNode(n v1.Node) (*NodeInfo, error) {
	nr := &NodeInfo{
		Name:     n.Name,
		Hostname: n.Labels[v1.LabelHostname],
	}

	for _, ip := range n.Status.Addresses {
		if ip.Type == v1.NodeExternalIP || ip.Type == v1.NodeInternalIP {
			logger.Debugf("using IP %s for node %s", ip.Address, n.Name)
			nr.Address = ip.Address
			break
		}
	}
	if nr.Address == "" {
		return nil, fmt.Errorf("couldn't get IP of node %s", nr.Name)
	}
	return nr, nil
}

// Returns a hierarchical representation of the nodes in the cluster organized
// by zone failure domain and annotated with the number of monitor pods assigned
// to each node and if the node is valid for scheduling monitor pods. The
// purpose of this structure is to provide a unified view of information
// required for making monitor scheduling decisions.
func (c *Cluster) getNodeMonUsage() ([][]NodeUsage, error) {
	// get all k8s node objects
	nodeOptions := metav1.ListOptions{}
	nodeOptions.TypeMeta.Kind = "Node"
	nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
	if err != nil {
		return nil, err
	}

	// get all pod objects labeled as a monitor
	podOptions := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}
	pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(podOptions)
	if err != nil {
		return nil, err
	}

	// generate a list of nodes that includes the number of monitor pods
	// assigned to each node. this is equivalent to:
	//
	//   SELECT Node, Count(*) FROM Nodes
	//   JOIN Pods ON Nodes.Hostname == Pods.Hostname
	//   GROUP BY Node
	//
	nodeUsages := []NodeUsage{}
	for i, node := range nodes.Items {
		valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
		if err != nil {
			logger.Warning("failed to validate node %s %v", node.Name, err)
			continue
		}
		nodeUsage := NodeUsage{Node: &nodes.Items[i], MonCount: 0, MonValid: valid}
		for _, pod := range pods.Items {
			hostname := pod.Spec.NodeSelector[v1.LabelHostname]
			if node.Name == hostname || node.Labels[v1.LabelHostname] == hostname {
				nodeUsage.MonCount++
			}
		}
		nodeUsages = append(nodeUsages, nodeUsage)
	}

	// generate a hierarchical representation of the nodes grouped by the name
	// of the node's zone failure domain label. the zone label named "" is used
	// for nodes without a zone annotation.
	nodesByZone := make(map[string][]NodeUsage)
	for _, nodeUsage := range nodeUsages {
		zone, ok := nodeUsage.Node.Labels["failure-domain.beta.kubernetes.io/zone"]
		if !ok {
			zone = ""
		}
		nodesByZone[zone] = append(nodesByZone[zone], nodeUsage)
	}

	// compute the final form used for scheduling. this form is an array of
	// nodes by zone, with non-labeled nodes appearing in the final zone. this
	// organization reflects the preference made by the scheduler of using nodes
	// with zone annotations before non-labeled nodes.
	res := [][]NodeUsage{}
	for zone, nodeUsage := range nodesByZone {
		if zone == "" {
			res = append(res, nodeUsage)
		} else {
			res = append([][]NodeUsage{nodeUsage}, res...)
		}
	}

	return res, nil
}