ceph: schedule mons across failure domains

this patch introduces failure-domain aware scheduling for monitor pods.
previously the only policy was to avoid scheduling monitors on the same
nodes. the new policy tries to spread monitors across zones and nodes.

   NOTE: this patch only enforces the well-known k8s "zone"
   failure-domain label (i.e. it does not look for region labels). this
   decision is made for two reasons. first, eliminating explicit
   scheduling is an important goal and will be done as part of upcoming
   work. this will enable all supported failure-domain labels to be
   handled. second, the "zone" domain is likely more appropriate for a
   larger portion of users (as opposed to the geographical distinction
   made with region), and supporting multiple failure domain types would
   add significant complexity for a temporary resolution to failure
   domain scheduling.

at a high-level this patch introduces a data structure `NodeUsage` that
is a pairing of a v1.Node and metadata relevant to scheduling: number of
monitor pods on the node, and its status w.r.t. to being scheduable for
new monitor pods.

each scheduling event computes a unified data structure that organizes
for each node a `NodeUsage` structure into per-failure-zone groups.
subsequent algorithms rely soley on this unified structure to make
scheduling decisions.

there are two entry points to the changes made:

1) during orchestration mon:Cluster:assignMons is invoked to make a
place new monitor pods onto nodes. the core scheduling logic is now
self-contained in mon:scheduleMonitor which implements node and zone
aware scheduling policies.

2) periodic health checks resolve any conflicts that may occur due to
changes to the cluster (e.g. new nodes, configuration changes, etc...).
the two existing checks: mons on invalid nodes, and overloaded nodes are
both handled.

fixes: #2603

Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
This commit is contained in:
Noah Watkins committed 2019-07-11 14:23:15 -07:00
1 parent 60cace8f7b
commit f034f000e1
9 files changed
+859 -248

No files matched your search

@@ -19,6 +19,7 @@ storage cluster.
- [OSD Dedicated Network](#osd-dedicated-network)
- [Phantom OSD Removal](#phantom-osd-removal)
- [Change Failure Domain](#change-failure-domain)
- [Monitor placement](#monitor-placement)
## Prerequisites
@@ -658,3 +659,23 @@ crush_rule: replicapool_host_rule
If the cluster's health was `HEALTH_OK` when we performed this change, immediately, the new rule is applied to the cluster transparently without service disruption.
Exactly the same approach can be used to change from `host` back to `osd`.
## Monitor placement
Rook will try to schedule Ceph monitor pods on different physical nodes by
default. When available, Rook will also consider failure domain information in
the form of Kubernetes node labels when scheduling Ceph monitors.
Currently Rook supports the node label
`failure-domain.beta.kubernetes.io/zone=<zone>` which can be applied to a node
to specify its failure domain using the command:
```
kubectl label node <node> failure-domain.beta.kubernetes.io/zone=<zone>
```
Rook uses failure domain labels by trying to schedule monitor pods on different
failure domains. And all nodes without failure domain labels are treated as a single
failure domain from a scheduling point of view. When placing multiple monitor
pods within a single failure domain Rook will try to run the pods on different
physical nodes.
+6
View File
@@ -22,6 +22,12 @@ an example usage
- `NodeAffinity` can be applied to `rook-ceph-agent DaemonSet` with `AGENT_NODE_AFFINITY` environment variable.
- `NodeAffinity` can be applied to `rook-discover DaemonSet` with `DISCOVER_AGENT_NODE_AFFINITY` environment variable.
- Rook does not create an initial CRUSH map anymore and let Ceph do it normally
- Ceph monitor placement will now take failure zones into account [see the
documentation](Documentation/ceph-advanced-configuration.md#monitor-placement)
for more information.
- The cluster CRD option to allow multiple monitors to be scheduled on the same
node---`spec.Mon.AllowMultiplePerNode`---is now active when a cluster is first
created. Previously, it was ignored when a cluster was first installed.
## Breaking Changes
+135 -57
View File
@@ -25,8 +25,10 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/client"
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
scheduler "k8s.io/kubernetes/pkg/scheduler/api"
)
var (
@@ -159,15 +161,9 @@ func (c *Cluster) checkHealth() error {
return nil
}
if !c.spec.Mon.AllowMultiplePerNode {
// check if there are more than two mons running on the same node, failover one mon in that case
done, err := c.checkMonsOnSameNode(desiredMonCount)
if done || err != nil {
return err
}
}
done, err := c.checkMonsOnValidNodes()
// find any mons that invalidate our placement policy, and if necessary,
// reschedule them to other nodes.
done, err := c.resolveInvalidMonitorPlacement(desiredMonCount)
if done || err != nil {
return err
}
@@ -191,54 +187,6 @@ func (c *Cluster) checkHealth() error {
return nil
}
func (c *Cluster) checkMonsOnSameNode(desiredMonCount int) (bool, error) {
nodesUsed := map[string]struct{}{}
for name, node := range c.mapping.Node {
// when the node is already in the list we have more than one mon on that node
if _, ok := nodesUsed[node.Name]; ok {
// get list of available nodes for mons
availableNodes, _, err := c.getAvailableMonNodes()
if err != nil {
return true, fmt.Errorf("failed to get available mon nodes. %+v", err)
}
// if there are enough nodes for one mon "that is too much" to be failovered,
// fail it over to an other node
if len(availableNodes) > 0 {
logger.Infof("rebalance: enough nodes available %d to failover mon %s", len(availableNodes), name)
c.failMon(len(c.clusterInfo.Monitors), desiredMonCount, name)
} else {
logger.Debugf("rebalance: not enough nodes available to failover mon %s", name)
}
// deal with one mon too much on a node at a time
return true, nil
}
nodesUsed[node.Name] = struct{}{}
}
return false, nil
}
func (c *Cluster) checkMonsOnValidNodes() (bool, error) {
for mon, nInfo := range c.mapping.Node {
// get node to use for validNode() func
node, err := c.context.Clientset.CoreV1().Nodes().Get(nInfo.Name, metav1.GetOptions{})
if err != nil {
return true, err
}
// check if node the mon is on is still valid
valid, err := k8sutil.ValidNode(*node, cephv1.GetMonPlacement(c.spec.Placement))
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
} else if !valid {
logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon)
c.failoverMon(mon)
return true, nil
}
logger.Debugf("node %s with mon %s is still valid", nInfo.Name, mon)
}
return false, nil
}
// failMon compares the monCount against desiredMonCount
func (c *Cluster) failMon(monCount, desiredMonCount int, name string) {
if monCount > desiredMonCount {
@@ -366,3 +314,133 @@ func removeMonitorFromQuorum(context *clusterd.Context, clusterName, name string
logger.Infof("removed monitor %s", name)
return nil
}
func (c *Cluster) resolveInvalidMonitorPlacement(desiredMonCount int) (bool, error) {
nodeChoice, err := c.findInvalidMonitorPlacement(desiredMonCount)
if err != nil {
return true, fmt.Errorf("failed to find invalid mon placement %+v", err)
}
// no violation found
if nodeChoice == nil {
return false, nil
}
for name, nodeInfo := range c.mapping.Node {
if nodeInfo.Name == nodeChoice.Node.Name {
logger.Infof("rebalance: rescheduling mon %s from node %s", name, nodeInfo.Name)
c.failMon(len(c.clusterInfo.Monitors), desiredMonCount, name)
return true, nil
}
}
logger.Warningf("rebalance: no mon pod found on node %s", nodeChoice.Node.Name)
return false, nil
}
func (c *Cluster) findInvalidMonitorPlacement(desiredMonCount int) (*NodeUsage, error) {
nodeZones, err := c.getNodeMonUsage()
if err != nil {
return nil, fmt.Errorf("failed to get node monitor usage. %+v", err)
}
// compute two helpful global flags:
// - does an empty zone exist
// - does an empty node exist
emptyZone := false
emptyNode := false
for zi := range nodeZones {
monFoundInZone := false
for ni := range nodeZones[zi] {
nodeUsage := &nodeZones[zi][ni]
if nodeUsage.MonCount > 0 {
monFoundInZone = true
} else if nodeUsage.MonValid {
// only consider valid nodes since this flag determines if a mon
// may be scheduled on to a new node.
emptyNode = true
}
}
if !monFoundInZone {
emptyZone = true
}
}
logger.Debugf("rebalance: desired mon count: %d, empty zone found: %t, empty node found: %t",
desiredMonCount, emptyZone, emptyNode)
for zi := range nodeZones {
zoneMonCount := 0
var nodeChoice *NodeUsage
// for each node consider two cases: too many monitors are running on a
// node, or monitors are running on invalid nodes.
for ni := range nodeZones[zi] {
nodeUsage := &nodeZones[zi][ni]
zoneMonCount += nodeUsage.MonCount
// if this node has too many monitors, and an underused node exists,
// then consider moving the monitor. note that this check is
// independent of the setting `AllowMultiplePerNode` since we also
// want to avoid in general keeping multiple monitors on one node.
if nodeUsage.MonCount > 1 && emptyNode {
logger.Infof("rebalance: chose overloaded node %s with %d mons",
nodeUsage.Node.Name, nodeUsage.MonCount)
nodeChoice = nodeUsage
break
}
// check for mons on invalid nodes. but reschedule pod only when it
// is invalid for reasons other than schedulability which should not
// affect pods that are already scheduled.
if nodeUsage.MonCount > 0 && !nodeUsage.MonValid {
placement := cephv1.GetMonPlacement(c.spec.Placement)
placement.Tolerations = append(placement.Tolerations,
v1.Toleration{
Key: scheduler.TaintNodeUnschedulable,
Effect: "",
Operator: v1.TolerationOpExists,
})
valid, err := k8sutil.ValidNodeNoSched(*nodeUsage.Node, placement)
if err != nil {
logger.Warningf("rebalance: failed to validate node %s %+v",
nodeUsage.Node.Name, err)
} else if !valid {
logger.Infof("rebalance: chose invalid node %s",
nodeUsage.Node.Name)
nodeChoice = nodeUsage
break
}
}
}
// if we didn't already select a node with a monitor for rescheduling,
// then consider that a zone may be overloaded and can be rebalanced.
// this case occurs if the zone has more than 1 monitor, and some other
// zone exists without any monitors.
if nodeChoice == nil && zoneMonCount > 1 && emptyZone {
for ni := range nodeZones[zi] {
nodeUsage := &nodeZones[zi][ni]
if nodeUsage.MonCount > 0 {
// select the node with the most monitors assigned
if nodeChoice == nil || nodeUsage.MonCount > nodeChoice.MonCount {
nodeChoice = nodeUsage
}
}
}
if nodeChoice != nil {
logger.Infof("rebalance: chose node %s from overloaded zone",
nodeChoice.Node.Name)
}
}
if nodeChoice != nil {
return nodeChoice, nil
}
}
logger.Debugf("rebalance: no mon placement violations or fixes available")
return nil, nil
}
+94 -7
View File
@@ -143,6 +143,91 @@ func TestCheckHealthNotFound(t *testing.T) {
}
}
func TestOverloadedZoneRebalances(t *testing.T) {
clientset := test.New(3)
// build a cluster with 3 nodes. two of the nodes are in the same AZ the
// third node is in a separate AZ.
for i := 0; i < 3; i++ {
nodeName := fmt.Sprintf("node%d", i)
node, err := clientset.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
// zone0: node0, node1
// zone1: node2
var zoneName string
if i < 2 {
zoneName = "zone0"
} else {
zoneName = "zone1"
}
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": zoneName}
clientset.CoreV1().Nodes().Update(node)
}
executor := &exectest.MockExecutor{}
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
context := &clusterd.Context{
Clientset: clientset,
ConfigDir: configDir,
Executor: executor,
}
cluster := New(context, "ns", "", false, metav1.OwnerReference{})
setCommonMonProperties(cluster, 2, cephv1.MonSpec{Count: 2, AllowMultiplePerNode: false}, "myversion")
cluster.waitForStart = false
defer os.RemoveAll(cluster.context.ConfigDir)
// add mon mappings
cluster.mapping.Node["a"] = &NodeInfo{
Name: "node0",
Hostname: "mynode0",
Address: "0.0.0.0",
}
cluster.mapping.Node["b"] = &NodeInfo{
Name: "node1",
Hostname: "mynode1",
Address: "0.0.0.0",
}
cluster.maxMonID = 1
cluster.saveMonConfig()
monNames := []string{"a", "b"}
nodeNames := []string{"node0", "node1"}
for i := 0; i < len(monNames); i++ {
monConfig := testGenMonConfig(monNames[i])
nodeName := nodeNames[i]
d := cluster.makeDeployment(monConfig, nodeName)
_, err := clientset.AppsV1().Deployments(cluster.Namespace).Create(d)
assert.Nil(t, err)
po := cluster.makeMonPod(monConfig, nodeName)
_, err = clientset.CoreV1().Pods(cluster.Namespace).Create(po)
assert.Nil(t, err)
}
assert.NotContains(t, cluster.mapping.Node, "c")
assert.Equal(t, "node0", cluster.mapping.Node["a"].Name)
assert.Equal(t, "node1", cluster.mapping.Node["b"].Name)
assert.Equal(t, "mynode0", cluster.mapping.Node["a"].Hostname)
assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname)
_, err := cluster.resolveInvalidMonitorPlacement(2)
assert.Nil(t, err)
// mon.a was chosen from the overloaded zone 0, and a new monitor to take
// its place, mon.c was created on a node in zone 1 that had no monitors.
assert.NotContains(t, cluster.mapping.Node, "a")
assert.Equal(t, "node1", cluster.mapping.Node["b"].Name)
assert.Equal(t, "node2", cluster.mapping.Node["c"].Name)
assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname)
}
func TestCheckHealthTwoMonsOneNode(t *testing.T) {
executorNextMons := false
executor := &exectest.MockExecutor{
@@ -207,7 +292,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) {
}
// initial health check should already see that there is more than one mon on one node (node0)
_, err := c.checkMonsOnSameNode(3)
_, err := c.resolveInvalidMonitorPlacement(3)
assert.Nil(t, err)
assert.Equal(t, "node0", c.mapping.Node["a"].Name)
assert.Equal(t, "node0", c.mapping.Node["b"].Name)
@@ -233,7 +318,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) {
n.Name = "node2"
clientset.CoreV1().Nodes().Create(n)
_, err = c.checkMonsOnSameNode(3)
_, err = c.resolveInvalidMonitorPlacement(3)
assert.Nil(t, err)
// check that mon c exists
@@ -255,7 +340,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) {
// enable different ceph mon map output
executorNextMons = true
_, err = c.checkMonsOnSameNode(3)
_, err = c.resolveInvalidMonitorPlacement(3)
assert.Nil(t, err)
// check that nothing has changed
@@ -322,15 +407,17 @@ func TestCheckMonsValid(t *testing.T) {
clientset.CoreV1().Nodes().Create(n)
}
_, err := c.checkMonsOnValidNodes()
_, err := c.resolveInvalidMonitorPlacement(3)
assert.Nil(t, err)
assert.Equal(t, "node0", c.mapping.Node["a"].Name)
assert.Equal(t, "node1", c.mapping.Node["b"].Name)
// set node1 unschedulable and check that mon2 gets failovered to be mon3 to node2
// set node0 to contain the not ready status condition, and check that mon.a
// is failovered to be mon.c on node2. note that validity here doesn't include the
// unscheduable flag (for mons already placed on a node).
node0, err := c.context.Clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
assert.Nil(t, err)
node0.Spec.Unschedulable = true
node0.Status.Conditions[0].Status = v1.ConditionFalse
_, err = c.context.Clientset.CoreV1().Nodes().Update(node0)
assert.Nil(t, err)
@@ -342,7 +429,7 @@ func TestCheckMonsValid(t *testing.T) {
assert.Nil(t, err)
}
_, err = c.checkMonsOnValidNodes()
_, err = c.resolveInvalidMonitorPlacement(3)
assert.Nil(t, err)
assert.Len(t, c.mapping.Node, 2)
+113 -21
View File
@@ -380,37 +380,129 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
return nil
}
func (c *Cluster) assignMons(mons []*monConfig) error {
// schedule the mons on different nodes if we have enough nodes to be unique
availableNodes, err := c.getMonNodes()
if err != nil {
return fmt.Errorf("failed to get available nodes for mons. %+v", err)
// find a suitable node for this monitor. first look for a fault zone that has
// no monitors. only after the nodes with zone labels are considered are the
// unlabeled nodes considered. reasonsing: without more information, _at worst_
// the unlabled nodes are actually in the set of labeled zones, and at best,
// they are in distinct zones.
func scheduleMonitor(mon *monConfig, nodeZones [][]NodeUsage) *NodeUsage {
// the node choice for this monitor
var nodeChoice *NodeUsage
// for each zone, in order of preference. unlabeled nodes are last, by
// construction; see Cluster.getNodeMonusage().
for zi := range nodeZones {
// number of monitors in the zone
zoneMonCount := 0
// preferential node choice from this zone for a new mon
var zoneNodeChoice *NodeUsage
// for each node in this zone
for ni := range nodeZones[zi] {
nodeUsage := &nodeZones[zi][ni]
// skip invalid nodes. but update the mon count first since
// invalid nodes could still have an assigned mon pod.
zoneMonCount += nodeUsage.MonCount
if !nodeUsage.MonValid {
logger.Infof("schedmon: skip invalid node %s for mon scheduling",
nodeUsage.Node.Name)
continue
}
// make a "best" choice from this zone. in this case that is the
// node with the least amount of monitors.
if zoneNodeChoice == nil || nodeUsage.MonCount < zoneNodeChoice.MonCount {
logger.Infof("schedmon: considering node %s with mon count %d",
nodeUsage.Node.Name, nodeUsage.MonCount)
zoneNodeChoice = nodeUsage
}
}
// We identified _a_ valid node in the zone. should we choose it?
if zoneNodeChoice != nil {
if zoneMonCount == 0 {
// this zone has no monitors, which implies that
// zoneNodeChoice will be a node without any monitors
// currently assigned. choose this node.
logger.Infof("schedmon: considering node %s from empty zone",
zoneNodeChoice.Node.Name)
nodeChoice = zoneNodeChoice
break
} else {
// this zone already has a monitor in it. but keep the
// choice as a backup; we may find that all zones already
// have a monitor and still need to make an assignment.
if nodeChoice == nil || zoneNodeChoice.MonCount < nodeChoice.MonCount {
logger.Infof("schedmon: considering node %s with mon count %d",
zoneNodeChoice.Node.Name, zoneNodeChoice.MonCount)
nodeChoice = zoneNodeChoice
}
}
}
}
nodeIndex := 0
for _, m := range mons {
if _, ok := c.mapping.Node[m.DaemonName]; ok {
logger.Debugf("mon %s already assigned to a node, no need to assign", m.DaemonName)
if nodeChoice != nil {
logger.Infof("schedmon: scheduling mon %s on node %s",
mon.DaemonName, nodeChoice.Node.Name)
} else {
logger.Infof("schedmon: no suitable node found for mon %s",
mon.DaemonName)
}
return nodeChoice
}
func (c *Cluster) assignMons(mons []*monConfig) error {
// retrieve the set of cluster nodes and their monitor usage info
nodeZones, err := c.getNodeMonUsage()
if err != nil {
return fmt.Errorf("failed to get node monitor usage. %+v", err)
}
// ensure all monitors have a node assignment. note that this isn't
// necessarily optimal: it does not try to move existing monitors which is
// handled by the periodic monitor health checks.
for _, mon := range mons {
// monitor is already assigned to a node. nothing to do
if _, ok := c.mapping.Node[mon.DaemonName]; ok {
logger.Infof("assignmon: mon %s already assigned to a node", mon.DaemonName)
continue
}
// if we need to place a new mon and don't have any more nodes available, we fail to add the mon
if len(availableNodes) == 0 {
return fmt.Errorf("no nodes available for mon placement")
nodeChoice := scheduleMonitor(mon, nodeZones)
if nodeChoice == nil {
return fmt.Errorf("assignmon: no valid nodes available for mon placement")
}
// pick one of the available nodes where the mon will be assigned
node := availableNodes[nodeIndex%len(availableNodes)]
logger.Debugf("mon %s assigned to node %s", m.DaemonName, node.Name)
nodeInfo, err := getNodeInfoFromNode(node)
if err != nil {
return fmt.Errorf("couldn't get node info from node %s. %+v", node.Name, err)
// note that we do not need to worry about a false negative here (i.e. a
// node exists that _would_ pass this check) due to the search landing in a
// local minima because if a node had existed with a mon count of zero,
// scheduleMonitor would have chosen it over any node with a positive
// mon count.
if nodeChoice.MonCount > 0 && !c.spec.Mon.AllowMultiplePerNode {
return fmt.Errorf("assignmon: no empty nodes available for mon placement")
}
c.mapping.Node[m.DaemonName] = nodeInfo
nodeIndex++
// make this decision visible when scheduling the next monitor
nodeChoice.MonCount++
logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Node.Name)
nodeInfo, err := getNodeInfoFromNode(*nodeChoice.Node)
if err != nil {
return fmt.Errorf("couldn't get node info from node %s. %+v",
nodeChoice.Node.Name, err)
}
c.mapping.Node[mon.DaemonName] = nodeInfo
}
logger.Debug("mons have been assigned to nodes")
logger.Debug("assignmons: mons have been assigned to nodes")
return nil
}
+165
View File
@@ -339,3 +339,168 @@ func TestMonFoundInQuorum(t *testing.T) {
assert.True(t, monFoundInQuorum("c", response))
assert.False(t, monFoundInQuorum("d", response))
}
// no node choice can be made when there are no nodes
func TestScheduleMonitorEmpty(t *testing.T) {
nodeZones := [][]NodeUsage{}
mon := &monConfig{DaemonName: "a"}
// no zones
assert.Nil(t, scheduleMonitor(mon, nodeZones))
// 1 zone no mons
nodeZones = append(nodeZones, []NodeUsage{})
assert.Nil(t, scheduleMonitor(mon, nodeZones))
// 2 zones no mons
nodeZones = append(nodeZones, []NodeUsage{})
assert.Nil(t, scheduleMonitor(mon, nodeZones))
}
// only valid nodes should be chosen
func TestScheduleMonitorInvalidNodes(t *testing.T) {
// 1 zone with 1 invalid empty node
nodeZones := [][]NodeUsage{
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
},
}
mon := &monConfig{DaemonName: "a"}
assert.Nil(t, scheduleMonitor(mon, nodeZones))
// 1 zone with 2 invalid empty node
nodeZones = [][]NodeUsage{
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
},
}
assert.Nil(t, scheduleMonitor(mon, nodeZones))
// 2 zone with 2 invalid empty node each
nodeZones = [][]NodeUsage{
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
},
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
},
}
assert.Nil(t, scheduleMonitor(mon, nodeZones))
}
func TestScheduleMonitor(t *testing.T) {
// 1 zone, 1 valid, empty node -> only one choice
nodeZones := [][]NodeUsage{
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
},
}
mon := &monConfig{DaemonName: "a"}
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
// still the only choice even if not empty
nodeZones = [][]NodeUsage{
{
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
},
}
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
// scheduler prefers the node with the least number of mons
nodeZones = [][]NodeUsage{
{
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
},
}
assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones))
// the scheduler prefers nodes with the least number of mons, not zones with
// the leads number of mons (zero mons is a special case: tested below...)
nodeZones = [][]NodeUsage{
// 24 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 4, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
},
// 10 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
},
}
// choose the node with 4 mons
assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones))
// same as before, the target mon is in the second zone
nodeZones = [][]NodeUsage{
// 6 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
},
// 10 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 9, MonValid: true},
},
}
// choose the node with 1 mon
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
// prefers a zone with zero mons to spread across failure domains
nodeZones = [][]NodeUsage{
// 0 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
},
// 1 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
},
}
// choose the zone with zero mons
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
// same as before, different zone
nodeZones = [][]NodeUsage{
// 0 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
},
// 1 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
},
}
// choose the zone with zero mons
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
// invalid nodes aren't schedulable, but if they have mons, that factors
// into the decision. in this case it means the zone isn't really empty
nodeZones = [][]NodeUsage{
// 0 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
// invalid node, but has a mon -> zone is not empty
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: false},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
},
// 1 mons
{
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
},
}
// choose the zone with zero mons
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
}
+82 -75
View File
@@ -21,85 +21,19 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// getMonNodes detects the nodes that are available for new mons to start.
func (c *Cluster) getMonNodes() ([]v1.Node, error) {
availableNodes, nodes, err := c.getAvailableMonNodes()
if err != nil {
return nil, err
}
logger.Infof("Found %d running nodes without mons", len(availableNodes))
// if all nodes already have mons and the user has given the mon.count, add all nodes to be available
if c.spec.Mon.AllowMultiplePerNode && len(availableNodes) == 0 {
logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items))
for _, node := range nodes.Items {
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
} else if valid {
availableNodes = append(availableNodes, node)
}
}
}
return availableNodes, nil
}
func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) {
nodeOptions := metav1.ListOptions{}
nodeOptions.TypeMeta.Kind = "Node"
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
if err != nil {
return nil, nil, err
}
logger.Debugf("there are %d nodes available for %d mons", len(nodes.Items), len(c.clusterInfo.Monitors))
// get the nodes that have mons assigned
nodesInUse, err := c.getNodesWithMons(nodes)
if err != nil {
logger.Warningf("could not get nodes with mons. %+v", err)
nodesInUse = util.NewSet()
}
// choose nodes for the new mons that don't have mons currently
availableNodes := []v1.Node{}
for _, node := range nodes.Items {
if !nodesInUse.Contains(node.Name) {
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
} else if valid {
availableNodes = append(availableNodes, node)
}
}
}
return availableNodes, nodes, nil
}
func (c *Cluster) getNodesWithMons(nodes *v1.NodeList) (*util.Set, error) {
// get the mon pods and their node affinity
options := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", appName)}
pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(options)
if err != nil {
return nil, err
}
nodesInUse := util.NewSet()
for _, pod := range pods.Items {
hostname := pod.Spec.NodeSelector[v1.LabelHostname]
logger.Debugf("mon pod on node %s", hostname)
name, ok := getNodeNameFromHostname(nodes, hostname)
if !ok {
logger.Errorf("mon %s on hostname %s not found in node list", pod.Name, hostname)
}
nodesInUse.Add(name)
}
return nodesInUse, nil
// NodeUsage is a mapping between a Node and computed metadata about the node
// that is used in monitor pod scheduling.
type NodeUsage struct {
Node *v1.Node
// The number of monitor pods assigned to the node
MonCount int
// The node is available for scheduling monitor pods. This is equivalent to
// evaluating k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
MonValid bool
}
// Get the number of mons that the operator should be starting
@@ -190,3 +124,76 @@ func getNodeInfoFromNode(n v1.Node) (*NodeInfo, error) {
}
return nr, nil
}
// Returns a hierarchical representation of the nodes in the cluster organized
// by zone failure domain and annotated with the number of monitor pods assigned
// to each node and if the node is valid for scheduling monitor pods. The
// purpose of this structure is to provide a unified view of information
// required for making monitor scheduling decisions.
func (c *Cluster) getNodeMonUsage() ([][]NodeUsage, error) {
// get all k8s node objects
nodeOptions := metav1.ListOptions{}
nodeOptions.TypeMeta.Kind = "Node"
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
if err != nil {
return nil, err
}
// get all pod objects labeled as a monitor
podOptions := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", appName)}
pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(podOptions)
if err != nil {
return nil, err
}
// generate a list of nodes that includes the number of monitor pods
// assigned to each node. this is equivalent to:
//
// SELECT Node, Count(*) FROM Nodes
// JOIN Pods ON Nodes.Hostname == Pods.Hostname
// GROUP BY Node
//
nodeUsages := []NodeUsage{}
for i, node := range nodes.Items {
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
continue
}
nodeUsage := NodeUsage{Node: &nodes.Items[i], MonCount: 0, MonValid: valid}
for _, pod := range pods.Items {
hostname := pod.Spec.NodeSelector[v1.LabelHostname]
if node.Name == hostname || node.Labels[v1.LabelHostname] == hostname {
nodeUsage.MonCount++
}
}
nodeUsages = append(nodeUsages, nodeUsage)
}
// generate a hierarchical representation of the nodes grouped by the name
// of the node's zone failure domain label. the zone label named "" is used
// for nodes without a zone annotation.
nodesByZone := make(map[string][]NodeUsage)
for _, nodeUsage := range nodeUsages {
zone, ok := nodeUsage.Node.Labels["failure-domain.beta.kubernetes.io/zone"]
if !ok {
zone = ""
}
nodesByZone[zone] = append(nodesByZone[zone], nodeUsage)
}
// compute the final form used for scheduling. this form is an array of
// nodes by zone, with non-labeled nodes appearing in the final zone. this
// organization reflects the preference made by the scheduler of using nodes
// with zone annotations before non-labeled nodes.
res := [][]NodeUsage{}
for zone, nodeUsage := range nodesByZone {
if zone == "" {
res = append(res, nodeUsage)
} else {
res = append([][]NodeUsage{nodeUsage}, res...)
}
}
return res, nil
}
+241 -87
View File
@@ -31,114 +31,115 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestAvailableMonNodes(t *testing.T) {
clientset := test.New(1)
func TestGetNodeMonUsageValidNode(t *testing.T) {
clientset := test.New(2)
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
nodes, err := c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 1, len(nodes))
// set node to not ready
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
conditions := v1.NodeCondition{Type: v1.NodeOutOfDisk}
nodes[0].Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}}
clientset.CoreV1().Nodes().Update(&nodes[0])
node.Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}}
clientset.CoreV1().Nodes().Update(node)
// when the node is not ready there should be no nodes returned and an error
emptyNodes, err := c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 0, len(emptyNodes))
// even if AllowMultiplePerNode is true there should be no node returned
c.spec.Mon.AllowMultiplePerNode = true
emptyNodes, err = c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 0, len(emptyNodes))
// nodes are reported with correct valid flag
nodeZones, err := c.getNodeMonUsage()
assert.NoError(t, err)
assert.False(t, nodeZones[0][0].MonValid)
assert.True(t, nodeZones[0][1].MonValid)
}
func TestAvailableNodesInUse(t *testing.T) {
clientset := test.New(3)
func TestGetNodeMonUsageMonCount(t *testing.T) {
clientset := test.New(2)
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
// all three nodes are available by default
nodes, err := c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 3, len(nodes))
// 3 mons on node0
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
// start pods on two of the nodes so that only one node will be available
monIDs := []string{"a", "b"}
for i := 0; i < 2; i++ {
pod := c.makeMonPod(testGenMonConfig(monIDs[i]), nodes[i].Name)
monIDs := []string{"a", "b", "c"}
for i := 0; i < 3; i++ {
pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name)
_, err = clientset.CoreV1().Pods(c.Namespace).Create(pod)
assert.Nil(t, err)
}
reducedNodes, err := c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 1, len(reducedNodes))
assert.Equal(t, nodes[2].Name, reducedNodes[0].Name)
// start pods on the remaining node. We expect no nodes to be available for placement
// since there is no way to place a mon on an unused node.
pod := c.makeMonPod(testGenMonConfig("c"), nodes[2].Name)
_, err = clientset.CoreV1().Pods(c.Namespace).Create(pod)
assert.Nil(t, err)
nodes, err = c.getMonNodes()
// no mon nodes is no error, just an empty nodes list
assert.Nil(t, err)
assert.Equal(t, 3, len(nodes))
// 2 mons on node1
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
// no nodes should be returned when AllowMultiplePerNode is false
c.spec.Mon.AllowMultiplePerNode = false
nodes, err = c.getMonNodes()
// no mon nodes is no error, just an empty nodes list
assert.Nil(t, err)
assert.Equal(t, 0, len(nodes))
monIDs = []string{"d", "e"}
for i := 0; i < 2; i++ {
pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name)
_, err = clientset.CoreV1().Pods(c.Namespace).Create(pod)
assert.Nil(t, err)
}
nodeZones, err := c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, 1, len(nodeZones))
if nodeZones[0][0].Node.Name == "node0" {
assert.Equal(t, 3, nodeZones[0][0].MonCount)
assert.Equal(t, 2, nodeZones[0][1].MonCount)
} else {
assert.Equal(t, 2, nodeZones[0][1].MonCount)
assert.Equal(t, 3, nodeZones[0][0].MonCount)
}
}
func TestTaintedNodes(t *testing.T) {
clientset := test.New(3)
clientset := test.New(4)
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
nodes, err := c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 3, len(nodes))
// mark a node as unschedulable
nodes[0].Spec.Unschedulable = true
clientset.CoreV1().Nodes().Update(&nodes[0])
nodesSchedulable, err := c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 2, len(nodesSchedulable))
nodes[0].Spec.Unschedulable = false
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
node.Spec.Unschedulable = true
clientset.CoreV1().Nodes().Update(node)
// taint nodes so they will not be schedulable for new pods
nodes[0].Spec.Taints = []v1.Taint{
// taint
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
node.Spec.Taints = []v1.Taint{
{Effect: v1.TaintEffectNoSchedule},
}
nodes[1].Spec.Taints = []v1.Taint{
clientset.CoreV1().Nodes().Update(node)
// taint
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
node.Spec.Taints = []v1.Taint{
{Effect: v1.TaintEffectPreferNoSchedule},
}
clientset.CoreV1().Nodes().Update(&nodes[0])
clientset.CoreV1().Nodes().Update(&nodes[1])
clientset.CoreV1().Nodes().Update(&nodes[2])
cleanNodes, err := c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 1, len(cleanNodes))
assert.Equal(t, nodes[2].Name, cleanNodes[0].Name)
clientset.CoreV1().Nodes().Update(node)
nodeZones, err := c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, 1, len(nodeZones))
validCount := 0
for zi := range nodeZones {
for ni := range nodeZones[zi] {
if nodeZones[zi][ni].MonValid {
validCount++
}
}
}
assert.Equal(t, 1, validCount)
}
func TestNodeAffinity(t *testing.T) {
clientset := test.New(3)
clientset := test.New(4)
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
nodes, err := c.getMonNodes()
assert.Nil(t, err)
assert.Equal(t, 3, len(nodes))
c.spec.Placement = map[rookalpha.KeyType]rookalpha.Placement{}
c.spec.Placement[cephv1.KeyMon] = rookalpha.Placement{NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
@@ -157,19 +158,31 @@ func TestNodeAffinity(t *testing.T) {
},
}
// label nodes so node0 will not be schedulable for new pods
nodes[0].Labels = map[string]string{"label": "foo"}
nodes[1].Labels = map[string]string{"label": "bar"}
nodes[2].Labels = map[string]string{"label": "baz"}
clientset.CoreV1().Nodes().Update(&nodes[0])
clientset.CoreV1().Nodes().Update(&nodes[1])
clientset.CoreV1().Nodes().Update(&nodes[2])
cleanNodes, err := c.getMonNodes()
// label nodes so they appear as not scheduable / invalid
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
node.Labels = map[string]string{"label": "foo"}
clientset.CoreV1().Nodes().Update(node)
assert.Nil(t, err)
assert.Equal(t, 2, len(cleanNodes))
assert.Equal(t, nodes[1].Name, cleanNodes[0].Name)
assert.Equal(t, nodes[2].Name, cleanNodes[1].Name)
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
node.Labels = map[string]string{"label": "bar"}
clientset.CoreV1().Nodes().Update(node)
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
node.Labels = map[string]string{"label": "baz"}
clientset.CoreV1().Nodes().Update(node)
nodeZones, err := c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, 1, len(nodeZones))
validCount := 0
for zi := range nodeZones {
for ni := range nodeZones[zi] {
if nodeZones[zi][ni].MonValid {
validCount++
}
}
}
assert.Equal(t, 2, validCount)
}
// this tests can 3 mons with hostnetworking on the same host is rejected
@@ -268,8 +281,18 @@ func TestHostNetwork(t *testing.T) {
c.HostNetwork = true
nodes, err := c.getMonNodes()
nodes := []v1.Node{}
nodeZones, err := c.getNodeMonUsage()
assert.Nil(t, err)
for zi := range nodeZones {
for _, nodeUsage := range nodeZones[zi] {
if nodeUsage.MonCount == 0 || c.spec.Mon.AllowMultiplePerNode {
nodes = append(nodes, *nodeUsage.Node)
}
}
}
assert.Equal(t, 3, len(nodes))
monConfig := testGenMonConfig("c")
@@ -375,3 +398,134 @@ func TestTargetMonCount(t *testing.T) {
assert.Equal(t, 7, target)
logger.Infof(msg)
}
// mon node usage should return no zones if there are no nodes
func TestGetNodeMonUsageNoNodes(t *testing.T) {
clientset := test.New(0)
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
nodeZones, err := c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, len(nodeZones), 0)
}
// all nodes without zone annotations are in the same zone
func TestGetNodeMonUsageNoZoneLabels(t *testing.T) {
for i := 1; i < 5; i++ {
clientset := test.New(i)
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
nodeZones, err := c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, len(nodeZones), 1)
assert.Equal(t, len(nodeZones[0]), i)
}
}
// nodes are partitioned into separate zones
func TestGetNodeMonUsageZoneSpread(t *testing.T) {
clientset := test.New(5)
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{})
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
// 1 node labeled -> 1 zone + the rest in the unlabeled zone
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"}
clientset.CoreV1().Nodes().Update(node)
// the zones are unlabeled and sort order isn't guaranteed. so instead we
// assert on expected count of zones and collection of zone sizes.
zoneCounts := map[int]int{}
nodeZones, err := c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, 2, len(nodeZones))
for _, zones := range nodeZones {
zoneSize := len(zones)
if _, ok := zoneCounts[zoneSize]; ok {
zoneCounts[zoneSize]++
} else {
zoneCounts[zoneSize] = 1
}
}
assert.Equal(t, map[int]int{1: 1, 4: 1}, zoneCounts)
// 2 nodes with same label, 3 unlabeled
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"}
clientset.CoreV1().Nodes().Update(node)
// reset
zoneCounts = map[int]int{}
nodeZones, err = c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, 2, len(nodeZones))
for _, zones := range nodeZones {
zoneSize := len(zones)
if _, ok := zoneCounts[zoneSize]; ok {
zoneCounts[zoneSize]++
} else {
zoneCounts[zoneSize] = 1
}
}
assert.Equal(t, map[int]int{2: 1, 3: 1}, zoneCounts)
// 2 nodes with same label, 2 with distinct labels, 1 unlabeled
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"}
clientset.CoreV1().Nodes().Update(node)
node, err = clientset.CoreV1().Nodes().Get("node3", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z2"}
clientset.CoreV1().Nodes().Update(node)
// reset
zoneCounts = map[int]int{}
nodeZones, err = c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, 4, len(nodeZones))
for _, zones := range nodeZones {
zoneSize := len(zones)
if _, ok := zoneCounts[zoneSize]; ok {
zoneCounts[zoneSize]++
} else {
zoneCounts[zoneSize] = 1
}
}
assert.Equal(t, map[int]int{2: 1, 1: 3}, zoneCounts)
// no unlabeled zones
node, err = clientset.CoreV1().Nodes().Get("node4", metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, node)
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"}
clientset.CoreV1().Nodes().Update(node)
// reset
zoneCounts = map[int]int{}
nodeZones, err = c.getNodeMonUsage()
assert.NoError(t, err)
assert.Equal(t, 3, len(nodeZones))
for _, zones := range nodeZones {
zoneSize := len(zones)
if _, ok := zoneCounts[zoneSize]; ok {
zoneCounts[zoneSize]++
} else {
zoneCounts[zoneSize] = 1
}
}
assert.Equal(t, map[int]int{2: 2, 1: 1}, zoneCounts)
}
+2 -1
View File
@@ -37,7 +37,8 @@ var validTopologyLabelKeys = []string{
}
// ValidNodeNoSched returns true if the node (1) meets Rook's placement terms,
// and (2) is ready. False otherwise.
// and (2) is ready. Unlike ValidNode, this method will ignore the
// Node.Spec.Unschedulable flag. False otherwise.
func ValidNodeNoSched(node v1.Node, placement rookalpha.Placement) (bool, error) {
p, err := NodeMeetsPlacementTerms(node, placement, false)
if err != nil {