ceph: schedule mons across failure domains
this patch introduces failure-domain aware scheduling for monitor pods. previously the only policy was to avoid scheduling monitors on the same nodes. the new policy tries to spread monitors across zones and nodes. NOTE: this patch only enforces the well-known k8s "zone" failure-domain label (i.e. it does not look for region labels). this decision is made for two reasons. first, eliminating explicit scheduling is an important goal and will be done as part of upcoming work. this will enable all supported failure-domain labels to be handled. second, the "zone" domain is likely more appropriate for a larger portion of users (as opposed to the geographical distinction made with region), and supporting multiple failure domain types would add significant complexity for a temporary resolution to failure domain scheduling. at a high-level this patch introduces a data structure `NodeUsage` that is a pairing of a v1.Node and metadata relevant to scheduling: number of monitor pods on the node, and its status w.r.t. to being scheduable for new monitor pods. each scheduling event computes a unified data structure that organizes for each node a `NodeUsage` structure into per-failure-zone groups. subsequent algorithms rely soley on this unified structure to make scheduling decisions. there are two entry points to the changes made: 1) during orchestration mon:Cluster:assignMons is invoked to make a place new monitor pods onto nodes. the core scheduling logic is now self-contained in mon:scheduleMonitor which implements node and zone aware scheduling policies. 2) periodic health checks resolve any conflicts that may occur due to changes to the cluster (e.g. new nodes, configuration changes, etc...). the two existing checks: mons on invalid nodes, and overloaded nodes are both handled. fixes: #2603 Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
This commit is contained in:
1 parent
60cace8f7b
commit
f034f000e1
9 files changed
+859
-248
No files matched your search
@@ -19,6 +19,7 @@ storage cluster.
|
||||
- [OSD Dedicated Network](#osd-dedicated-network)
|
||||
- [Phantom OSD Removal](#phantom-osd-removal)
|
||||
- [Change Failure Domain](#change-failure-domain)
|
||||
- [Monitor placement](#monitor-placement)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -658,3 +659,23 @@ crush_rule: replicapool_host_rule
|
||||
If the cluster's health was `HEALTH_OK` when we performed this change, immediately, the new rule is applied to the cluster transparently without service disruption.
|
||||
|
||||
Exactly the same approach can be used to change from `host` back to `osd`.
|
||||
|
||||
## Monitor placement
|
||||
|
||||
Rook will try to schedule Ceph monitor pods on different physical nodes by
|
||||
default. When available, Rook will also consider failure domain information in
|
||||
the form of Kubernetes node labels when scheduling Ceph monitors.
|
||||
|
||||
Currently Rook supports the node label
|
||||
`failure-domain.beta.kubernetes.io/zone=<zone>` which can be applied to a node
|
||||
to specify its failure domain using the command:
|
||||
|
||||
```
|
||||
kubectl label node <node> failure-domain.beta.kubernetes.io/zone=<zone>
|
||||
```
|
||||
|
||||
Rook uses failure domain labels by trying to schedule monitor pods on different
|
||||
failure domains. And all nodes without failure domain labels are treated as a single
|
||||
failure domain from a scheduling point of view. When placing multiple monitor
|
||||
pods within a single failure domain Rook will try to run the pods on different
|
||||
physical nodes.
|
||||
@@ -22,6 +22,12 @@ an example usage
|
||||
- `NodeAffinity` can be applied to `rook-ceph-agent DaemonSet` with `AGENT_NODE_AFFINITY` environment variable.
|
||||
- `NodeAffinity` can be applied to `rook-discover DaemonSet` with `DISCOVER_AGENT_NODE_AFFINITY` environment variable.
|
||||
- Rook does not create an initial CRUSH map anymore and let Ceph do it normally
|
||||
- Ceph monitor placement will now take failure zones into account [see the
|
||||
documentation](Documentation/ceph-advanced-configuration.md#monitor-placement)
|
||||
for more information.
|
||||
- The cluster CRD option to allow multiple monitors to be scheduled on the same
|
||||
node---`spec.Mon.AllowMultiplePerNode`---is now active when a cluster is first
|
||||
created. Previously, it was ignored when a cluster was first installed.
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
|
||||
@@ -25,8 +25,10 @@ import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
scheduler "k8s.io/kubernetes/pkg/scheduler/api"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -159,15 +161,9 @@ func (c *Cluster) checkHealth() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !c.spec.Mon.AllowMultiplePerNode {
|
||||
// check if there are more than two mons running on the same node, failover one mon in that case
|
||||
done, err := c.checkMonsOnSameNode(desiredMonCount)
|
||||
if done || err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
done, err := c.checkMonsOnValidNodes()
|
||||
// find any mons that invalidate our placement policy, and if necessary,
|
||||
// reschedule them to other nodes.
|
||||
done, err := c.resolveInvalidMonitorPlacement(desiredMonCount)
|
||||
if done || err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -191,54 +187,6 @@ func (c *Cluster) checkHealth() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) checkMonsOnSameNode(desiredMonCount int) (bool, error) {
|
||||
nodesUsed := map[string]struct{}{}
|
||||
for name, node := range c.mapping.Node {
|
||||
// when the node is already in the list we have more than one mon on that node
|
||||
if _, ok := nodesUsed[node.Name]; ok {
|
||||
// get list of available nodes for mons
|
||||
availableNodes, _, err := c.getAvailableMonNodes()
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("failed to get available mon nodes. %+v", err)
|
||||
}
|
||||
// if there are enough nodes for one mon "that is too much" to be failovered,
|
||||
// fail it over to an other node
|
||||
if len(availableNodes) > 0 {
|
||||
logger.Infof("rebalance: enough nodes available %d to failover mon %s", len(availableNodes), name)
|
||||
c.failMon(len(c.clusterInfo.Monitors), desiredMonCount, name)
|
||||
} else {
|
||||
logger.Debugf("rebalance: not enough nodes available to failover mon %s", name)
|
||||
}
|
||||
|
||||
// deal with one mon too much on a node at a time
|
||||
return true, nil
|
||||
}
|
||||
nodesUsed[node.Name] = struct{}{}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) checkMonsOnValidNodes() (bool, error) {
|
||||
for mon, nInfo := range c.mapping.Node {
|
||||
// get node to use for validNode() func
|
||||
node, err := c.context.Clientset.CoreV1().Nodes().Get(nInfo.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
// check if node the mon is on is still valid
|
||||
valid, err := k8sutil.ValidNode(*node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if !valid {
|
||||
logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon)
|
||||
c.failoverMon(mon)
|
||||
return true, nil
|
||||
}
|
||||
logger.Debugf("node %s with mon %s is still valid", nInfo.Name, mon)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// failMon compares the monCount against desiredMonCount
|
||||
func (c *Cluster) failMon(monCount, desiredMonCount int, name string) {
|
||||
if monCount > desiredMonCount {
|
||||
@@ -366,3 +314,133 @@ func removeMonitorFromQuorum(context *clusterd.Context, clusterName, name string
|
||||
logger.Infof("removed monitor %s", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) resolveInvalidMonitorPlacement(desiredMonCount int) (bool, error) {
|
||||
nodeChoice, err := c.findInvalidMonitorPlacement(desiredMonCount)
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("failed to find invalid mon placement %+v", err)
|
||||
}
|
||||
|
||||
// no violation found
|
||||
if nodeChoice == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for name, nodeInfo := range c.mapping.Node {
|
||||
if nodeInfo.Name == nodeChoice.Node.Name {
|
||||
logger.Infof("rebalance: rescheduling mon %s from node %s", name, nodeInfo.Name)
|
||||
c.failMon(len(c.clusterInfo.Monitors), desiredMonCount, name)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
logger.Warningf("rebalance: no mon pod found on node %s", nodeChoice.Node.Name)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) findInvalidMonitorPlacement(desiredMonCount int) (*NodeUsage, error) {
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get node monitor usage. %+v", err)
|
||||
}
|
||||
|
||||
// compute two helpful global flags:
|
||||
// - does an empty zone exist
|
||||
// - does an empty node exist
|
||||
emptyZone := false
|
||||
emptyNode := false
|
||||
for zi := range nodeZones {
|
||||
monFoundInZone := false
|
||||
for ni := range nodeZones[zi] {
|
||||
nodeUsage := &nodeZones[zi][ni]
|
||||
if nodeUsage.MonCount > 0 {
|
||||
monFoundInZone = true
|
||||
} else if nodeUsage.MonValid {
|
||||
// only consider valid nodes since this flag determines if a mon
|
||||
// may be scheduled on to a new node.
|
||||
emptyNode = true
|
||||
}
|
||||
}
|
||||
if !monFoundInZone {
|
||||
emptyZone = true
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("rebalance: desired mon count: %d, empty zone found: %t, empty node found: %t",
|
||||
desiredMonCount, emptyZone, emptyNode)
|
||||
|
||||
for zi := range nodeZones {
|
||||
zoneMonCount := 0
|
||||
var nodeChoice *NodeUsage
|
||||
|
||||
// for each node consider two cases: too many monitors are running on a
|
||||
// node, or monitors are running on invalid nodes.
|
||||
for ni := range nodeZones[zi] {
|
||||
nodeUsage := &nodeZones[zi][ni]
|
||||
zoneMonCount += nodeUsage.MonCount
|
||||
|
||||
// if this node has too many monitors, and an underused node exists,
|
||||
// then consider moving the monitor. note that this check is
|
||||
// independent of the setting `AllowMultiplePerNode` since we also
|
||||
// want to avoid in general keeping multiple monitors on one node.
|
||||
if nodeUsage.MonCount > 1 && emptyNode {
|
||||
logger.Infof("rebalance: chose overloaded node %s with %d mons",
|
||||
nodeUsage.Node.Name, nodeUsage.MonCount)
|
||||
nodeChoice = nodeUsage
|
||||
break
|
||||
}
|
||||
|
||||
// check for mons on invalid nodes. but reschedule pod only when it
|
||||
// is invalid for reasons other than schedulability which should not
|
||||
// affect pods that are already scheduled.
|
||||
if nodeUsage.MonCount > 0 && !nodeUsage.MonValid {
|
||||
placement := cephv1.GetMonPlacement(c.spec.Placement)
|
||||
placement.Tolerations = append(placement.Tolerations,
|
||||
v1.Toleration{
|
||||
Key: scheduler.TaintNodeUnschedulable,
|
||||
Effect: "",
|
||||
Operator: v1.TolerationOpExists,
|
||||
})
|
||||
valid, err := k8sutil.ValidNodeNoSched(*nodeUsage.Node, placement)
|
||||
if err != nil {
|
||||
logger.Warningf("rebalance: failed to validate node %s %+v",
|
||||
nodeUsage.Node.Name, err)
|
||||
} else if !valid {
|
||||
logger.Infof("rebalance: chose invalid node %s",
|
||||
nodeUsage.Node.Name)
|
||||
nodeChoice = nodeUsage
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we didn't already select a node with a monitor for rescheduling,
|
||||
// then consider that a zone may be overloaded and can be rebalanced.
|
||||
// this case occurs if the zone has more than 1 monitor, and some other
|
||||
// zone exists without any monitors.
|
||||
if nodeChoice == nil && zoneMonCount > 1 && emptyZone {
|
||||
for ni := range nodeZones[zi] {
|
||||
nodeUsage := &nodeZones[zi][ni]
|
||||
if nodeUsage.MonCount > 0 {
|
||||
// select the node with the most monitors assigned
|
||||
if nodeChoice == nil || nodeUsage.MonCount > nodeChoice.MonCount {
|
||||
nodeChoice = nodeUsage
|
||||
}
|
||||
}
|
||||
}
|
||||
if nodeChoice != nil {
|
||||
logger.Infof("rebalance: chose node %s from overloaded zone",
|
||||
nodeChoice.Node.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if nodeChoice != nil {
|
||||
return nodeChoice, nil
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("rebalance: no mon placement violations or fixes available")
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
@@ -143,6 +143,91 @@ func TestCheckHealthNotFound(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverloadedZoneRebalances(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
|
||||
// build a cluster with 3 nodes. two of the nodes are in the same AZ the
|
||||
// third node is in a separate AZ.
|
||||
for i := 0; i < 3; i++ {
|
||||
nodeName := fmt.Sprintf("node%d", i)
|
||||
node, err := clientset.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
|
||||
// zone0: node0, node1
|
||||
// zone1: node2
|
||||
var zoneName string
|
||||
if i < 2 {
|
||||
zoneName = "zone0"
|
||||
} else {
|
||||
zoneName = "zone1"
|
||||
}
|
||||
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": zoneName}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
}
|
||||
|
||||
executor := &exectest.MockExecutor{}
|
||||
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
context := &clusterd.Context{
|
||||
Clientset: clientset,
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
|
||||
cluster := New(context, "ns", "", false, metav1.OwnerReference{})
|
||||
setCommonMonProperties(cluster, 2, cephv1.MonSpec{Count: 2, AllowMultiplePerNode: false}, "myversion")
|
||||
cluster.waitForStart = false
|
||||
defer os.RemoveAll(cluster.context.ConfigDir)
|
||||
|
||||
// add mon mappings
|
||||
cluster.mapping.Node["a"] = &NodeInfo{
|
||||
Name: "node0",
|
||||
Hostname: "mynode0",
|
||||
Address: "0.0.0.0",
|
||||
}
|
||||
cluster.mapping.Node["b"] = &NodeInfo{
|
||||
Name: "node1",
|
||||
Hostname: "mynode1",
|
||||
Address: "0.0.0.0",
|
||||
}
|
||||
cluster.maxMonID = 1
|
||||
cluster.saveMonConfig()
|
||||
|
||||
monNames := []string{"a", "b"}
|
||||
nodeNames := []string{"node0", "node1"}
|
||||
for i := 0; i < len(monNames); i++ {
|
||||
monConfig := testGenMonConfig(monNames[i])
|
||||
nodeName := nodeNames[i]
|
||||
|
||||
d := cluster.makeDeployment(monConfig, nodeName)
|
||||
_, err := clientset.AppsV1().Deployments(cluster.Namespace).Create(d)
|
||||
assert.Nil(t, err)
|
||||
|
||||
po := cluster.makeMonPod(monConfig, nodeName)
|
||||
_, err = clientset.CoreV1().Pods(cluster.Namespace).Create(po)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
assert.NotContains(t, cluster.mapping.Node, "c")
|
||||
assert.Equal(t, "node0", cluster.mapping.Node["a"].Name)
|
||||
assert.Equal(t, "node1", cluster.mapping.Node["b"].Name)
|
||||
assert.Equal(t, "mynode0", cluster.mapping.Node["a"].Hostname)
|
||||
assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname)
|
||||
|
||||
_, err := cluster.resolveInvalidMonitorPlacement(2)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// mon.a was chosen from the overloaded zone 0, and a new monitor to take
|
||||
// its place, mon.c was created on a node in zone 1 that had no monitors.
|
||||
assert.NotContains(t, cluster.mapping.Node, "a")
|
||||
assert.Equal(t, "node1", cluster.mapping.Node["b"].Name)
|
||||
assert.Equal(t, "node2", cluster.mapping.Node["c"].Name)
|
||||
assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname)
|
||||
}
|
||||
|
||||
func TestCheckHealthTwoMonsOneNode(t *testing.T) {
|
||||
executorNextMons := false
|
||||
executor := &exectest.MockExecutor{
|
||||
@@ -207,7 +292,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) {
|
||||
}
|
||||
|
||||
// initial health check should already see that there is more than one mon on one node (node0)
|
||||
_, err := c.checkMonsOnSameNode(3)
|
||||
_, err := c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "node0", c.mapping.Node["a"].Name)
|
||||
assert.Equal(t, "node0", c.mapping.Node["b"].Name)
|
||||
@@ -233,7 +318,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) {
|
||||
n.Name = "node2"
|
||||
clientset.CoreV1().Nodes().Create(n)
|
||||
|
||||
_, err = c.checkMonsOnSameNode(3)
|
||||
_, err = c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// check that mon c exists
|
||||
@@ -255,7 +340,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) {
|
||||
|
||||
// enable different ceph mon map output
|
||||
executorNextMons = true
|
||||
_, err = c.checkMonsOnSameNode(3)
|
||||
_, err = c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// check that nothing has changed
|
||||
@@ -322,15 +407,17 @@ func TestCheckMonsValid(t *testing.T) {
|
||||
clientset.CoreV1().Nodes().Create(n)
|
||||
}
|
||||
|
||||
_, err := c.checkMonsOnValidNodes()
|
||||
_, err := c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "node0", c.mapping.Node["a"].Name)
|
||||
assert.Equal(t, "node1", c.mapping.Node["b"].Name)
|
||||
|
||||
// set node1 unschedulable and check that mon2 gets failovered to be mon3 to node2
|
||||
// set node0 to contain the not ready status condition, and check that mon.a
|
||||
// is failovered to be mon.c on node2. note that validity here doesn't include the
|
||||
// unscheduable flag (for mons already placed on a node).
|
||||
node0, err := c.context.Clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
node0.Spec.Unschedulable = true
|
||||
node0.Status.Conditions[0].Status = v1.ConditionFalse
|
||||
_, err = c.context.Clientset.CoreV1().Nodes().Update(node0)
|
||||
assert.Nil(t, err)
|
||||
|
||||
@@ -342,7 +429,7 @@ func TestCheckMonsValid(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
_, err = c.checkMonsOnValidNodes()
|
||||
_, err = c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Len(t, c.mapping.Node, 2)
|
||||
|
||||
@@ -380,37 +380,129 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
// schedule the mons on different nodes if we have enough nodes to be unique
|
||||
availableNodes, err := c.getMonNodes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get available nodes for mons. %+v", err)
|
||||
// find a suitable node for this monitor. first look for a fault zone that has
|
||||
// no monitors. only after the nodes with zone labels are considered are the
|
||||
// unlabeled nodes considered. reasonsing: without more information, _at worst_
|
||||
// the unlabled nodes are actually in the set of labeled zones, and at best,
|
||||
// they are in distinct zones.
|
||||
func scheduleMonitor(mon *monConfig, nodeZones [][]NodeUsage) *NodeUsage {
|
||||
// the node choice for this monitor
|
||||
var nodeChoice *NodeUsage
|
||||
|
||||
// for each zone, in order of preference. unlabeled nodes are last, by
|
||||
// construction; see Cluster.getNodeMonusage().
|
||||
for zi := range nodeZones {
|
||||
// number of monitors in the zone
|
||||
zoneMonCount := 0
|
||||
|
||||
// preferential node choice from this zone for a new mon
|
||||
var zoneNodeChoice *NodeUsage
|
||||
|
||||
// for each node in this zone
|
||||
for ni := range nodeZones[zi] {
|
||||
nodeUsage := &nodeZones[zi][ni]
|
||||
|
||||
// skip invalid nodes. but update the mon count first since
|
||||
// invalid nodes could still have an assigned mon pod.
|
||||
zoneMonCount += nodeUsage.MonCount
|
||||
if !nodeUsage.MonValid {
|
||||
logger.Infof("schedmon: skip invalid node %s for mon scheduling",
|
||||
nodeUsage.Node.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
// make a "best" choice from this zone. in this case that is the
|
||||
// node with the least amount of monitors.
|
||||
if zoneNodeChoice == nil || nodeUsage.MonCount < zoneNodeChoice.MonCount {
|
||||
logger.Infof("schedmon: considering node %s with mon count %d",
|
||||
nodeUsage.Node.Name, nodeUsage.MonCount)
|
||||
zoneNodeChoice = nodeUsage
|
||||
}
|
||||
}
|
||||
|
||||
// We identified _a_ valid node in the zone. should we choose it?
|
||||
if zoneNodeChoice != nil {
|
||||
if zoneMonCount == 0 {
|
||||
// this zone has no monitors, which implies that
|
||||
// zoneNodeChoice will be a node without any monitors
|
||||
// currently assigned. choose this node.
|
||||
logger.Infof("schedmon: considering node %s from empty zone",
|
||||
zoneNodeChoice.Node.Name)
|
||||
nodeChoice = zoneNodeChoice
|
||||
break
|
||||
} else {
|
||||
// this zone already has a monitor in it. but keep the
|
||||
// choice as a backup; we may find that all zones already
|
||||
// have a monitor and still need to make an assignment.
|
||||
if nodeChoice == nil || zoneNodeChoice.MonCount < nodeChoice.MonCount {
|
||||
logger.Infof("schedmon: considering node %s with mon count %d",
|
||||
zoneNodeChoice.Node.Name, zoneNodeChoice.MonCount)
|
||||
nodeChoice = zoneNodeChoice
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nodeIndex := 0
|
||||
for _, m := range mons {
|
||||
if _, ok := c.mapping.Node[m.DaemonName]; ok {
|
||||
logger.Debugf("mon %s already assigned to a node, no need to assign", m.DaemonName)
|
||||
if nodeChoice != nil {
|
||||
logger.Infof("schedmon: scheduling mon %s on node %s",
|
||||
mon.DaemonName, nodeChoice.Node.Name)
|
||||
} else {
|
||||
logger.Infof("schedmon: no suitable node found for mon %s",
|
||||
mon.DaemonName)
|
||||
}
|
||||
|
||||
return nodeChoice
|
||||
}
|
||||
|
||||
func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
|
||||
// retrieve the set of cluster nodes and their monitor usage info
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get node monitor usage. %+v", err)
|
||||
}
|
||||
|
||||
// ensure all monitors have a node assignment. note that this isn't
|
||||
// necessarily optimal: it does not try to move existing monitors which is
|
||||
// handled by the periodic monitor health checks.
|
||||
for _, mon := range mons {
|
||||
|
||||
// monitor is already assigned to a node. nothing to do
|
||||
if _, ok := c.mapping.Node[mon.DaemonName]; ok {
|
||||
logger.Infof("assignmon: mon %s already assigned to a node", mon.DaemonName)
|
||||
continue
|
||||
}
|
||||
|
||||
// if we need to place a new mon and don't have any more nodes available, we fail to add the mon
|
||||
if len(availableNodes) == 0 {
|
||||
return fmt.Errorf("no nodes available for mon placement")
|
||||
nodeChoice := scheduleMonitor(mon, nodeZones)
|
||||
|
||||
if nodeChoice == nil {
|
||||
return fmt.Errorf("assignmon: no valid nodes available for mon placement")
|
||||
}
|
||||
|
||||
// pick one of the available nodes where the mon will be assigned
|
||||
node := availableNodes[nodeIndex%len(availableNodes)]
|
||||
logger.Debugf("mon %s assigned to node %s", m.DaemonName, node.Name)
|
||||
nodeInfo, err := getNodeInfoFromNode(node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't get node info from node %s. %+v", node.Name, err)
|
||||
// note that we do not need to worry about a false negative here (i.e. a
|
||||
// node exists that _would_ pass this check) due to the search landing in a
|
||||
// local minima because if a node had existed with a mon count of zero,
|
||||
// scheduleMonitor would have chosen it over any node with a positive
|
||||
// mon count.
|
||||
if nodeChoice.MonCount > 0 && !c.spec.Mon.AllowMultiplePerNode {
|
||||
return fmt.Errorf("assignmon: no empty nodes available for mon placement")
|
||||
}
|
||||
c.mapping.Node[m.DaemonName] = nodeInfo
|
||||
nodeIndex++
|
||||
|
||||
// make this decision visible when scheduling the next monitor
|
||||
nodeChoice.MonCount++
|
||||
|
||||
logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Node.Name)
|
||||
|
||||
nodeInfo, err := getNodeInfoFromNode(*nodeChoice.Node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't get node info from node %s. %+v",
|
||||
nodeChoice.Node.Name, err)
|
||||
}
|
||||
|
||||
c.mapping.Node[mon.DaemonName] = nodeInfo
|
||||
}
|
||||
|
||||
logger.Debug("mons have been assigned to nodes")
|
||||
logger.Debug("assignmons: mons have been assigned to nodes")
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -339,3 +339,168 @@ func TestMonFoundInQuorum(t *testing.T) {
|
||||
assert.True(t, monFoundInQuorum("c", response))
|
||||
assert.False(t, monFoundInQuorum("d", response))
|
||||
}
|
||||
|
||||
// no node choice can be made when there are no nodes
|
||||
func TestScheduleMonitorEmpty(t *testing.T) {
|
||||
nodeZones := [][]NodeUsage{}
|
||||
mon := &monConfig{DaemonName: "a"}
|
||||
// no zones
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
// 1 zone no mons
|
||||
nodeZones = append(nodeZones, []NodeUsage{})
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
// 2 zones no mons
|
||||
nodeZones = append(nodeZones, []NodeUsage{})
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
}
|
||||
|
||||
// only valid nodes should be chosen
|
||||
func TestScheduleMonitorInvalidNodes(t *testing.T) {
|
||||
// 1 zone with 1 invalid empty node
|
||||
nodeZones := [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
},
|
||||
}
|
||||
mon := &monConfig{DaemonName: "a"}
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// 1 zone with 2 invalid empty node
|
||||
nodeZones = [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
},
|
||||
}
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// 2 zone with 2 invalid empty node each
|
||||
nodeZones = [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
},
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
},
|
||||
}
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
}
|
||||
|
||||
func TestScheduleMonitor(t *testing.T) {
|
||||
// 1 zone, 1 valid, empty node -> only one choice
|
||||
nodeZones := [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
}
|
||||
mon := &monConfig{DaemonName: "a"}
|
||||
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// still the only choice even if not empty
|
||||
nodeZones = [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
|
||||
},
|
||||
}
|
||||
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// scheduler prefers the node with the least number of mons
|
||||
nodeZones = [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
|
||||
},
|
||||
}
|
||||
assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// the scheduler prefers nodes with the least number of mons, not zones with
|
||||
// the leads number of mons (zero mons is a special case: tested below...)
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 24 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 4, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
|
||||
},
|
||||
// 10 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the node with 4 mons
|
||||
assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// same as before, the target mon is in the second zone
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 6 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
|
||||
},
|
||||
// 10 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 9, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the node with 1 mon
|
||||
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// prefers a zone with zero mons to spread across failure domains
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 0 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
// 1 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the zone with zero mons
|
||||
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// same as before, different zone
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 0 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
// 1 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the zone with zero mons
|
||||
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// invalid nodes aren't schedulable, but if they have mons, that factors
|
||||
// into the decision. in this case it means the zone isn't really empty
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 0 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
// invalid node, but has a mon -> zone is not empty
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: false},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
// 1 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the zone with zero mons
|
||||
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
|
||||
}
|
||||
@@ -21,85 +21,19 @@ import (
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// getMonNodes detects the nodes that are available for new mons to start.
|
||||
func (c *Cluster) getMonNodes() ([]v1.Node, error) {
|
||||
availableNodes, nodes, err := c.getAvailableMonNodes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger.Infof("Found %d running nodes without mons", len(availableNodes))
|
||||
|
||||
// if all nodes already have mons and the user has given the mon.count, add all nodes to be available
|
||||
if c.spec.Mon.AllowMultiplePerNode && len(availableNodes) == 0 {
|
||||
logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items))
|
||||
for _, node := range nodes.Items {
|
||||
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
availableNodes = append(availableNodes, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return availableNodes, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) {
|
||||
nodeOptions := metav1.ListOptions{}
|
||||
nodeOptions.TypeMeta.Kind = "Node"
|
||||
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
logger.Debugf("there are %d nodes available for %d mons", len(nodes.Items), len(c.clusterInfo.Monitors))
|
||||
|
||||
// get the nodes that have mons assigned
|
||||
nodesInUse, err := c.getNodesWithMons(nodes)
|
||||
if err != nil {
|
||||
logger.Warningf("could not get nodes with mons. %+v", err)
|
||||
nodesInUse = util.NewSet()
|
||||
}
|
||||
|
||||
// choose nodes for the new mons that don't have mons currently
|
||||
availableNodes := []v1.Node{}
|
||||
for _, node := range nodes.Items {
|
||||
if !nodesInUse.Contains(node.Name) {
|
||||
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
availableNodes = append(availableNodes, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return availableNodes, nodes, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) getNodesWithMons(nodes *v1.NodeList) (*util.Set, error) {
|
||||
// get the mon pods and their node affinity
|
||||
options := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", appName)}
|
||||
pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodesInUse := util.NewSet()
|
||||
for _, pod := range pods.Items {
|
||||
hostname := pod.Spec.NodeSelector[v1.LabelHostname]
|
||||
logger.Debugf("mon pod on node %s", hostname)
|
||||
name, ok := getNodeNameFromHostname(nodes, hostname)
|
||||
if !ok {
|
||||
logger.Errorf("mon %s on hostname %s not found in node list", pod.Name, hostname)
|
||||
}
|
||||
nodesInUse.Add(name)
|
||||
}
|
||||
return nodesInUse, nil
|
||||
// NodeUsage is a mapping between a Node and computed metadata about the node
|
||||
// that is used in monitor pod scheduling.
|
||||
type NodeUsage struct {
|
||||
Node *v1.Node
|
||||
// The number of monitor pods assigned to the node
|
||||
MonCount int
|
||||
// The node is available for scheduling monitor pods. This is equivalent to
|
||||
// evaluating k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
MonValid bool
|
||||
}
|
||||
|
||||
// Get the number of mons that the operator should be starting
|
||||
@@ -190,3 +124,76 @@ func getNodeInfoFromNode(n v1.Node) (*NodeInfo, error) {
|
||||
}
|
||||
return nr, nil
|
||||
}
|
||||
|
||||
// Returns a hierarchical representation of the nodes in the cluster organized
|
||||
// by zone failure domain and annotated with the number of monitor pods assigned
|
||||
// to each node and if the node is valid for scheduling monitor pods. The
|
||||
// purpose of this structure is to provide a unified view of information
|
||||
// required for making monitor scheduling decisions.
|
||||
func (c *Cluster) getNodeMonUsage() ([][]NodeUsage, error) {
|
||||
// get all k8s node objects
|
||||
nodeOptions := metav1.ListOptions{}
|
||||
nodeOptions.TypeMeta.Kind = "Node"
|
||||
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// get all pod objects labeled as a monitor
|
||||
podOptions := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", appName)}
|
||||
pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(podOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// generate a list of nodes that includes the number of monitor pods
|
||||
// assigned to each node. this is equivalent to:
|
||||
//
|
||||
// SELECT Node, Count(*) FROM Nodes
|
||||
// JOIN Pods ON Nodes.Hostname == Pods.Hostname
|
||||
// GROUP BY Node
|
||||
//
|
||||
nodeUsages := []NodeUsage{}
|
||||
for i, node := range nodes.Items {
|
||||
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
continue
|
||||
}
|
||||
nodeUsage := NodeUsage{Node: &nodes.Items[i], MonCount: 0, MonValid: valid}
|
||||
for _, pod := range pods.Items {
|
||||
hostname := pod.Spec.NodeSelector[v1.LabelHostname]
|
||||
if node.Name == hostname || node.Labels[v1.LabelHostname] == hostname {
|
||||
nodeUsage.MonCount++
|
||||
}
|
||||
}
|
||||
nodeUsages = append(nodeUsages, nodeUsage)
|
||||
}
|
||||
|
||||
// generate a hierarchical representation of the nodes grouped by the name
|
||||
// of the node's zone failure domain label. the zone label named "" is used
|
||||
// for nodes without a zone annotation.
|
||||
nodesByZone := make(map[string][]NodeUsage)
|
||||
for _, nodeUsage := range nodeUsages {
|
||||
zone, ok := nodeUsage.Node.Labels["failure-domain.beta.kubernetes.io/zone"]
|
||||
if !ok {
|
||||
zone = ""
|
||||
}
|
||||
nodesByZone[zone] = append(nodesByZone[zone], nodeUsage)
|
||||
}
|
||||
|
||||
// compute the final form used for scheduling. this form is an array of
|
||||
// nodes by zone, with non-labeled nodes appearing in the final zone. this
|
||||
// organization reflects the preference made by the scheduler of using nodes
|
||||
// with zone annotations before non-labeled nodes.
|
||||
res := [][]NodeUsage{}
|
||||
for zone, nodeUsage := range nodesByZone {
|
||||
if zone == "" {
|
||||
res = append(res, nodeUsage)
|
||||
} else {
|
||||
res = append([][]NodeUsage{nodeUsage}, res...)
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
@@ -31,114 +31,115 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestAvailableMonNodes(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
func TestGetNodeMonUsageValidNode(t *testing.T) {
|
||||
clientset := test.New(2)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
nodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(nodes))
|
||||
|
||||
// set node to not ready
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
conditions := v1.NodeCondition{Type: v1.NodeOutOfDisk}
|
||||
nodes[0].Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}}
|
||||
clientset.CoreV1().Nodes().Update(&nodes[0])
|
||||
node.Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// when the node is not ready there should be no nodes returned and an error
|
||||
emptyNodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(emptyNodes))
|
||||
|
||||
// even if AllowMultiplePerNode is true there should be no node returned
|
||||
c.spec.Mon.AllowMultiplePerNode = true
|
||||
emptyNodes, err = c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(emptyNodes))
|
||||
// nodes are reported with correct valid flag
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, nodeZones[0][0].MonValid)
|
||||
assert.True(t, nodeZones[0][1].MonValid)
|
||||
}
|
||||
|
||||
func TestAvailableNodesInUse(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
func TestGetNodeMonUsageMonCount(t *testing.T) {
|
||||
clientset := test.New(2)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
// all three nodes are available by default
|
||||
nodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, len(nodes))
|
||||
// 3 mons on node0
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
|
||||
// start pods on two of the nodes so that only one node will be available
|
||||
monIDs := []string{"a", "b"}
|
||||
for i := 0; i < 2; i++ {
|
||||
pod := c.makeMonPod(testGenMonConfig(monIDs[i]), nodes[i].Name)
|
||||
monIDs := []string{"a", "b", "c"}
|
||||
for i := 0; i < 3; i++ {
|
||||
pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name)
|
||||
_, err = clientset.CoreV1().Pods(c.Namespace).Create(pod)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
reducedNodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(reducedNodes))
|
||||
assert.Equal(t, nodes[2].Name, reducedNodes[0].Name)
|
||||
|
||||
// start pods on the remaining node. We expect no nodes to be available for placement
|
||||
// since there is no way to place a mon on an unused node.
|
||||
pod := c.makeMonPod(testGenMonConfig("c"), nodes[2].Name)
|
||||
_, err = clientset.CoreV1().Pods(c.Namespace).Create(pod)
|
||||
assert.Nil(t, err)
|
||||
nodes, err = c.getMonNodes()
|
||||
// no mon nodes is no error, just an empty nodes list
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, len(nodes))
|
||||
// 2 mons on node1
|
||||
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
|
||||
// no nodes should be returned when AllowMultiplePerNode is false
|
||||
c.spec.Mon.AllowMultiplePerNode = false
|
||||
nodes, err = c.getMonNodes()
|
||||
// no mon nodes is no error, just an empty nodes list
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(nodes))
|
||||
monIDs = []string{"d", "e"}
|
||||
for i := 0; i < 2; i++ {
|
||||
pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name)
|
||||
_, err = clientset.CoreV1().Pods(c.Namespace).Create(pod)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(nodeZones))
|
||||
if nodeZones[0][0].Node.Name == "node0" {
|
||||
assert.Equal(t, 3, nodeZones[0][0].MonCount)
|
||||
assert.Equal(t, 2, nodeZones[0][1].MonCount)
|
||||
} else {
|
||||
assert.Equal(t, 2, nodeZones[0][1].MonCount)
|
||||
assert.Equal(t, 3, nodeZones[0][0].MonCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaintedNodes(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
clientset := test.New(4)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
nodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, len(nodes))
|
||||
|
||||
// mark a node as unschedulable
|
||||
nodes[0].Spec.Unschedulable = true
|
||||
clientset.CoreV1().Nodes().Update(&nodes[0])
|
||||
nodesSchedulable, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(nodesSchedulable))
|
||||
nodes[0].Spec.Unschedulable = false
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Spec.Unschedulable = true
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// taint nodes so they will not be schedulable for new pods
|
||||
nodes[0].Spec.Taints = []v1.Taint{
|
||||
// taint
|
||||
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Spec.Taints = []v1.Taint{
|
||||
{Effect: v1.TaintEffectNoSchedule},
|
||||
}
|
||||
nodes[1].Spec.Taints = []v1.Taint{
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// taint
|
||||
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Spec.Taints = []v1.Taint{
|
||||
{Effect: v1.TaintEffectPreferNoSchedule},
|
||||
}
|
||||
clientset.CoreV1().Nodes().Update(&nodes[0])
|
||||
clientset.CoreV1().Nodes().Update(&nodes[1])
|
||||
clientset.CoreV1().Nodes().Update(&nodes[2])
|
||||
cleanNodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(cleanNodes))
|
||||
assert.Equal(t, nodes[2].Name, cleanNodes[0].Name)
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(nodeZones))
|
||||
validCount := 0
|
||||
for zi := range nodeZones {
|
||||
for ni := range nodeZones[zi] {
|
||||
if nodeZones[zi][ni].MonValid {
|
||||
validCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 1, validCount)
|
||||
}
|
||||
|
||||
func TestNodeAffinity(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
clientset := test.New(4)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
nodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, len(nodes))
|
||||
|
||||
c.spec.Placement = map[rookalpha.KeyType]rookalpha.Placement{}
|
||||
c.spec.Placement[cephv1.KeyMon] = rookalpha.Placement{NodeAffinity: &v1.NodeAffinity{
|
||||
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
|
||||
@@ -157,19 +158,31 @@ func TestNodeAffinity(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
// label nodes so node0 will not be schedulable for new pods
|
||||
nodes[0].Labels = map[string]string{"label": "foo"}
|
||||
nodes[1].Labels = map[string]string{"label": "bar"}
|
||||
nodes[2].Labels = map[string]string{"label": "baz"}
|
||||
clientset.CoreV1().Nodes().Update(&nodes[0])
|
||||
clientset.CoreV1().Nodes().Update(&nodes[1])
|
||||
clientset.CoreV1().Nodes().Update(&nodes[2])
|
||||
cleanNodes, err := c.getMonNodes()
|
||||
// label nodes so they appear as not scheduable / invalid
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
node.Labels = map[string]string{"label": "foo"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(cleanNodes))
|
||||
assert.Equal(t, nodes[1].Name, cleanNodes[0].Name)
|
||||
assert.Equal(t, nodes[2].Name, cleanNodes[1].Name)
|
||||
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
node.Labels = map[string]string{"label": "bar"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
|
||||
node.Labels = map[string]string{"label": "baz"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(nodeZones))
|
||||
validCount := 0
|
||||
for zi := range nodeZones {
|
||||
for ni := range nodeZones[zi] {
|
||||
if nodeZones[zi][ni].MonValid {
|
||||
validCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 2, validCount)
|
||||
}
|
||||
|
||||
// this tests can 3 mons with hostnetworking on the same host is rejected
|
||||
@@ -268,8 +281,18 @@ func TestHostNetwork(t *testing.T) {
|
||||
|
||||
c.HostNetwork = true
|
||||
|
||||
nodes, err := c.getMonNodes()
|
||||
nodes := []v1.Node{}
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.Nil(t, err)
|
||||
|
||||
for zi := range nodeZones {
|
||||
for _, nodeUsage := range nodeZones[zi] {
|
||||
if nodeUsage.MonCount == 0 || c.spec.Mon.AllowMultiplePerNode {
|
||||
nodes = append(nodes, *nodeUsage.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, 3, len(nodes))
|
||||
|
||||
monConfig := testGenMonConfig("c")
|
||||
@@ -375,3 +398,134 @@ func TestTargetMonCount(t *testing.T) {
|
||||
assert.Equal(t, 7, target)
|
||||
logger.Infof(msg)
|
||||
}
|
||||
|
||||
// mon node usage should return no zones if there are no nodes
|
||||
func TestGetNodeMonUsageNoNodes(t *testing.T) {
|
||||
clientset := test.New(0)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(nodeZones), 0)
|
||||
}
|
||||
|
||||
// all nodes without zone annotations are in the same zone
|
||||
func TestGetNodeMonUsageNoZoneLabels(t *testing.T) {
|
||||
for i := 1; i < 5; i++ {
|
||||
clientset := test.New(i)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(nodeZones), 1)
|
||||
assert.Equal(t, len(nodeZones[0]), i)
|
||||
}
|
||||
}
|
||||
|
||||
// nodes are partitioned into separate zones
|
||||
func TestGetNodeMonUsageZoneSpread(t *testing.T) {
|
||||
clientset := test.New(5)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
// 1 node labeled -> 1 zone + the rest in the unlabeled zone
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// the zones are unlabeled and sort order isn't guaranteed. so instead we
|
||||
// assert on expected count of zones and collection of zone sizes.
|
||||
zoneCounts := map[int]int{}
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, len(nodeZones))
|
||||
for _, zones := range nodeZones {
|
||||
zoneSize := len(zones)
|
||||
if _, ok := zoneCounts[zoneSize]; ok {
|
||||
zoneCounts[zoneSize]++
|
||||
} else {
|
||||
zoneCounts[zoneSize] = 1
|
||||
}
|
||||
}
|
||||
assert.Equal(t, map[int]int{1: 1, 4: 1}, zoneCounts)
|
||||
|
||||
// 2 nodes with same label, 3 unlabeled
|
||||
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// reset
|
||||
zoneCounts = map[int]int{}
|
||||
|
||||
nodeZones, err = c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, len(nodeZones))
|
||||
for _, zones := range nodeZones {
|
||||
zoneSize := len(zones)
|
||||
if _, ok := zoneCounts[zoneSize]; ok {
|
||||
zoneCounts[zoneSize]++
|
||||
} else {
|
||||
zoneCounts[zoneSize] = 1
|
||||
}
|
||||
}
|
||||
assert.Equal(t, map[int]int{2: 1, 3: 1}, zoneCounts)
|
||||
|
||||
// 2 nodes with same label, 2 with distinct labels, 1 unlabeled
|
||||
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
node, err = clientset.CoreV1().Nodes().Get("node3", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z2"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// reset
|
||||
zoneCounts = map[int]int{}
|
||||
|
||||
nodeZones, err = c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 4, len(nodeZones))
|
||||
for _, zones := range nodeZones {
|
||||
zoneSize := len(zones)
|
||||
if _, ok := zoneCounts[zoneSize]; ok {
|
||||
zoneCounts[zoneSize]++
|
||||
} else {
|
||||
zoneCounts[zoneSize] = 1
|
||||
}
|
||||
}
|
||||
assert.Equal(t, map[int]int{2: 1, 1: 3}, zoneCounts)
|
||||
|
||||
// no unlabeled zones
|
||||
node, err = clientset.CoreV1().Nodes().Get("node4", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// reset
|
||||
zoneCounts = map[int]int{}
|
||||
|
||||
nodeZones, err = c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, len(nodeZones))
|
||||
for _, zones := range nodeZones {
|
||||
zoneSize := len(zones)
|
||||
if _, ok := zoneCounts[zoneSize]; ok {
|
||||
zoneCounts[zoneSize]++
|
||||
} else {
|
||||
zoneCounts[zoneSize] = 1
|
||||
}
|
||||
}
|
||||
assert.Equal(t, map[int]int{2: 2, 1: 1}, zoneCounts)
|
||||
}
|
||||
@@ -37,7 +37,8 @@ var validTopologyLabelKeys = []string{
|
||||
}
|
||||
|
||||
// ValidNodeNoSched returns true if the node (1) meets Rook's placement terms,
|
||||
// and (2) is ready. False otherwise.
|
||||
// and (2) is ready. Unlike ValidNode, this method will ignore the
|
||||
// Node.Spec.Unschedulable flag. False otherwise.
|
||||
func ValidNodeNoSched(node v1.Node, placement rookalpha.Placement) (bool, error) {
|
||||
p, err := NodeMeetsPlacementTerms(node, placement, false)
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user