diff --git a/Documentation/ceph-advanced-configuration.md b/Documentation/ceph-advanced-configuration.md index c1a906be7..5712872da 100644 --- a/Documentation/ceph-advanced-configuration.md +++ b/Documentation/ceph-advanced-configuration.md @@ -19,6 +19,7 @@ storage cluster. - [OSD Dedicated Network](#osd-dedicated-network) - [Phantom OSD Removal](#phantom-osd-removal) - [Change Failure Domain](#change-failure-domain) +- [Monitor placement](#monitor-placement) ## Prerequisites @@ -658,3 +659,23 @@ crush_rule: replicapool_host_rule If the cluster's health was `HEALTH_OK` when we performed this change, immediately, the new rule is applied to the cluster transparently without service disruption. Exactly the same approach can be used to change from `host` back to `osd`. + +## Monitor placement + +Rook will try to schedule Ceph monitor pods on different physical nodes by +default. When available, Rook will also consider failure domain information in +the form of Kubernetes node labels when scheduling Ceph monitors. + +Currently Rook supports the node label +`failure-domain.beta.kubernetes.io/zone=` which can be applied to a node +to specify its failure domain using the command: + +``` +kubectl label node failure-domain.beta.kubernetes.io/zone= +``` + +Rook uses failure domain labels by trying to schedule monitor pods on different +failure domains. And all nodes without failure domain labels are treated as a single +failure domain from a scheduling point of view. When placing multiple monitor +pods within a single failure domain Rook will try to run the pods on different +physical nodes. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 799c6a962..3124fd4ab 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -22,6 +22,12 @@ an example usage - `NodeAffinity` can be applied to `rook-ceph-agent DaemonSet` with `AGENT_NODE_AFFINITY` environment variable. - `NodeAffinity` can be applied to `rook-discover DaemonSet` with `DISCOVER_AGENT_NODE_AFFINITY` environment variable. - Rook does not create an initial CRUSH map anymore and let Ceph do it normally +- Ceph monitor placement will now take failure zones into account [see the + documentation](Documentation/ceph-advanced-configuration.md#monitor-placement) + for more information. +- The cluster CRD option to allow multiple monitors to be scheduled on the same + node---`spec.Mon.AllowMultiplePerNode`---is now active when a cluster is first + created. Previously, it was ignored when a cluster was first installed. ## Breaking Changes diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 2389b7110..6d672b735 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -25,8 +25,10 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + scheduler "k8s.io/kubernetes/pkg/scheduler/api" ) var ( @@ -159,15 +161,9 @@ func (c *Cluster) checkHealth() error { return nil } - if !c.spec.Mon.AllowMultiplePerNode { - // check if there are more than two mons running on the same node, failover one mon in that case - done, err := c.checkMonsOnSameNode(desiredMonCount) - if done || err != nil { - return err - } - } - - done, err := c.checkMonsOnValidNodes() + // find any mons that invalidate our placement policy, and if necessary, + // reschedule them to other nodes. + done, err := c.resolveInvalidMonitorPlacement(desiredMonCount) if done || err != nil { return err } @@ -191,54 +187,6 @@ func (c *Cluster) checkHealth() error { return nil } -func (c *Cluster) checkMonsOnSameNode(desiredMonCount int) (bool, error) { - nodesUsed := map[string]struct{}{} - for name, node := range c.mapping.Node { - // when the node is already in the list we have more than one mon on that node - if _, ok := nodesUsed[node.Name]; ok { - // get list of available nodes for mons - availableNodes, _, err := c.getAvailableMonNodes() - if err != nil { - return true, fmt.Errorf("failed to get available mon nodes. %+v", err) - } - // if there are enough nodes for one mon "that is too much" to be failovered, - // fail it over to an other node - if len(availableNodes) > 0 { - logger.Infof("rebalance: enough nodes available %d to failover mon %s", len(availableNodes), name) - c.failMon(len(c.clusterInfo.Monitors), desiredMonCount, name) - } else { - logger.Debugf("rebalance: not enough nodes available to failover mon %s", name) - } - - // deal with one mon too much on a node at a time - return true, nil - } - nodesUsed[node.Name] = struct{}{} - } - return false, nil -} - -func (c *Cluster) checkMonsOnValidNodes() (bool, error) { - for mon, nInfo := range c.mapping.Node { - // get node to use for validNode() func - node, err := c.context.Clientset.CoreV1().Nodes().Get(nInfo.Name, metav1.GetOptions{}) - if err != nil { - return true, err - } - // check if node the mon is on is still valid - valid, err := k8sutil.ValidNode(*node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - } else if !valid { - logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon) - c.failoverMon(mon) - return true, nil - } - logger.Debugf("node %s with mon %s is still valid", nInfo.Name, mon) - } - return false, nil -} - // failMon compares the monCount against desiredMonCount func (c *Cluster) failMon(monCount, desiredMonCount int, name string) { if monCount > desiredMonCount { @@ -366,3 +314,133 @@ func removeMonitorFromQuorum(context *clusterd.Context, clusterName, name string logger.Infof("removed monitor %s", name) return nil } + +func (c *Cluster) resolveInvalidMonitorPlacement(desiredMonCount int) (bool, error) { + nodeChoice, err := c.findInvalidMonitorPlacement(desiredMonCount) + if err != nil { + return true, fmt.Errorf("failed to find invalid mon placement %+v", err) + } + + // no violation found + if nodeChoice == nil { + return false, nil + } + + for name, nodeInfo := range c.mapping.Node { + if nodeInfo.Name == nodeChoice.Node.Name { + logger.Infof("rebalance: rescheduling mon %s from node %s", name, nodeInfo.Name) + c.failMon(len(c.clusterInfo.Monitors), desiredMonCount, name) + return true, nil + } + } + + logger.Warningf("rebalance: no mon pod found on node %s", nodeChoice.Node.Name) + + return false, nil +} + +func (c *Cluster) findInvalidMonitorPlacement(desiredMonCount int) (*NodeUsage, error) { + nodeZones, err := c.getNodeMonUsage() + if err != nil { + return nil, fmt.Errorf("failed to get node monitor usage. %+v", err) + } + + // compute two helpful global flags: + // - does an empty zone exist + // - does an empty node exist + emptyZone := false + emptyNode := false + for zi := range nodeZones { + monFoundInZone := false + for ni := range nodeZones[zi] { + nodeUsage := &nodeZones[zi][ni] + if nodeUsage.MonCount > 0 { + monFoundInZone = true + } else if nodeUsage.MonValid { + // only consider valid nodes since this flag determines if a mon + // may be scheduled on to a new node. + emptyNode = true + } + } + if !monFoundInZone { + emptyZone = true + } + } + + logger.Debugf("rebalance: desired mon count: %d, empty zone found: %t, empty node found: %t", + desiredMonCount, emptyZone, emptyNode) + + for zi := range nodeZones { + zoneMonCount := 0 + var nodeChoice *NodeUsage + + // for each node consider two cases: too many monitors are running on a + // node, or monitors are running on invalid nodes. + for ni := range nodeZones[zi] { + nodeUsage := &nodeZones[zi][ni] + zoneMonCount += nodeUsage.MonCount + + // if this node has too many monitors, and an underused node exists, + // then consider moving the monitor. note that this check is + // independent of the setting `AllowMultiplePerNode` since we also + // want to avoid in general keeping multiple monitors on one node. + if nodeUsage.MonCount > 1 && emptyNode { + logger.Infof("rebalance: chose overloaded node %s with %d mons", + nodeUsage.Node.Name, nodeUsage.MonCount) + nodeChoice = nodeUsage + break + } + + // check for mons on invalid nodes. but reschedule pod only when it + // is invalid for reasons other than schedulability which should not + // affect pods that are already scheduled. + if nodeUsage.MonCount > 0 && !nodeUsage.MonValid { + placement := cephv1.GetMonPlacement(c.spec.Placement) + placement.Tolerations = append(placement.Tolerations, + v1.Toleration{ + Key: scheduler.TaintNodeUnschedulable, + Effect: "", + Operator: v1.TolerationOpExists, + }) + valid, err := k8sutil.ValidNodeNoSched(*nodeUsage.Node, placement) + if err != nil { + logger.Warningf("rebalance: failed to validate node %s %+v", + nodeUsage.Node.Name, err) + } else if !valid { + logger.Infof("rebalance: chose invalid node %s", + nodeUsage.Node.Name) + nodeChoice = nodeUsage + break + } + } + } + + // if we didn't already select a node with a monitor for rescheduling, + // then consider that a zone may be overloaded and can be rebalanced. + // this case occurs if the zone has more than 1 monitor, and some other + // zone exists without any monitors. + if nodeChoice == nil && zoneMonCount > 1 && emptyZone { + for ni := range nodeZones[zi] { + nodeUsage := &nodeZones[zi][ni] + if nodeUsage.MonCount > 0 { + // select the node with the most monitors assigned + if nodeChoice == nil || nodeUsage.MonCount > nodeChoice.MonCount { + nodeChoice = nodeUsage + } + } + } + if nodeChoice != nil { + logger.Infof("rebalance: chose node %s from overloaded zone", + nodeChoice.Node.Name) + } + } + + if nodeChoice != nil { + return nodeChoice, nil + } + } + + logger.Debugf("rebalance: no mon placement violations or fixes available") + + return nil, nil +} diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 7099dc0ac..4b1b2091b 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -143,6 +143,91 @@ func TestCheckHealthNotFound(t *testing.T) { } } +func TestOverloadedZoneRebalances(t *testing.T) { + clientset := test.New(3) + + // build a cluster with 3 nodes. two of the nodes are in the same AZ the + // third node is in a separate AZ. + for i := 0; i < 3; i++ { + nodeName := fmt.Sprintf("node%d", i) + node, err := clientset.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + + // zone0: node0, node1 + // zone1: node2 + var zoneName string + if i < 2 { + zoneName = "zone0" + } else { + zoneName = "zone1" + } + + node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": zoneName} + clientset.CoreV1().Nodes().Update(node) + } + + executor := &exectest.MockExecutor{} + + configDir, _ := ioutil.TempDir("", "") + defer os.RemoveAll(configDir) + context := &clusterd.Context{ + Clientset: clientset, + ConfigDir: configDir, + Executor: executor, + } + + cluster := New(context, "ns", "", false, metav1.OwnerReference{}) + setCommonMonProperties(cluster, 2, cephv1.MonSpec{Count: 2, AllowMultiplePerNode: false}, "myversion") + cluster.waitForStart = false + defer os.RemoveAll(cluster.context.ConfigDir) + + // add mon mappings + cluster.mapping.Node["a"] = &NodeInfo{ + Name: "node0", + Hostname: "mynode0", + Address: "0.0.0.0", + } + cluster.mapping.Node["b"] = &NodeInfo{ + Name: "node1", + Hostname: "mynode1", + Address: "0.0.0.0", + } + cluster.maxMonID = 1 + cluster.saveMonConfig() + + monNames := []string{"a", "b"} + nodeNames := []string{"node0", "node1"} + for i := 0; i < len(monNames); i++ { + monConfig := testGenMonConfig(monNames[i]) + nodeName := nodeNames[i] + + d := cluster.makeDeployment(monConfig, nodeName) + _, err := clientset.AppsV1().Deployments(cluster.Namespace).Create(d) + assert.Nil(t, err) + + po := cluster.makeMonPod(monConfig, nodeName) + _, err = clientset.CoreV1().Pods(cluster.Namespace).Create(po) + assert.Nil(t, err) + } + + assert.NotContains(t, cluster.mapping.Node, "c") + assert.Equal(t, "node0", cluster.mapping.Node["a"].Name) + assert.Equal(t, "node1", cluster.mapping.Node["b"].Name) + assert.Equal(t, "mynode0", cluster.mapping.Node["a"].Hostname) + assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname) + + _, err := cluster.resolveInvalidMonitorPlacement(2) + assert.Nil(t, err) + + // mon.a was chosen from the overloaded zone 0, and a new monitor to take + // its place, mon.c was created on a node in zone 1 that had no monitors. + assert.NotContains(t, cluster.mapping.Node, "a") + assert.Equal(t, "node1", cluster.mapping.Node["b"].Name) + assert.Equal(t, "node2", cluster.mapping.Node["c"].Name) + assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname) +} + func TestCheckHealthTwoMonsOneNode(t *testing.T) { executorNextMons := false executor := &exectest.MockExecutor{ @@ -207,7 +292,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) { } // initial health check should already see that there is more than one mon on one node (node0) - _, err := c.checkMonsOnSameNode(3) + _, err := c.resolveInvalidMonitorPlacement(3) assert.Nil(t, err) assert.Equal(t, "node0", c.mapping.Node["a"].Name) assert.Equal(t, "node0", c.mapping.Node["b"].Name) @@ -233,7 +318,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) { n.Name = "node2" clientset.CoreV1().Nodes().Create(n) - _, err = c.checkMonsOnSameNode(3) + _, err = c.resolveInvalidMonitorPlacement(3) assert.Nil(t, err) // check that mon c exists @@ -255,7 +340,7 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) { // enable different ceph mon map output executorNextMons = true - _, err = c.checkMonsOnSameNode(3) + _, err = c.resolveInvalidMonitorPlacement(3) assert.Nil(t, err) // check that nothing has changed @@ -322,15 +407,17 @@ func TestCheckMonsValid(t *testing.T) { clientset.CoreV1().Nodes().Create(n) } - _, err := c.checkMonsOnValidNodes() + _, err := c.resolveInvalidMonitorPlacement(3) assert.Nil(t, err) assert.Equal(t, "node0", c.mapping.Node["a"].Name) assert.Equal(t, "node1", c.mapping.Node["b"].Name) - // set node1 unschedulable and check that mon2 gets failovered to be mon3 to node2 + // set node0 to contain the not ready status condition, and check that mon.a + // is failovered to be mon.c on node2. note that validity here doesn't include the + // unscheduable flag (for mons already placed on a node). node0, err := c.context.Clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) assert.Nil(t, err) - node0.Spec.Unschedulable = true + node0.Status.Conditions[0].Status = v1.ConditionFalse _, err = c.context.Clientset.CoreV1().Nodes().Update(node0) assert.Nil(t, err) @@ -342,7 +429,7 @@ func TestCheckMonsValid(t *testing.T) { assert.Nil(t, err) } - _, err = c.checkMonsOnValidNodes() + _, err = c.resolveInvalidMonitorPlacement(3) assert.Nil(t, err) assert.Len(t, c.mapping.Node, 2) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index b436a2926..f715624e2 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -380,37 +380,129 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error { return nil } -func (c *Cluster) assignMons(mons []*monConfig) error { - // schedule the mons on different nodes if we have enough nodes to be unique - availableNodes, err := c.getMonNodes() - if err != nil { - return fmt.Errorf("failed to get available nodes for mons. %+v", err) +// find a suitable node for this monitor. first look for a fault zone that has +// no monitors. only after the nodes with zone labels are considered are the +// unlabeled nodes considered. reasonsing: without more information, _at worst_ +// the unlabled nodes are actually in the set of labeled zones, and at best, +// they are in distinct zones. +func scheduleMonitor(mon *monConfig, nodeZones [][]NodeUsage) *NodeUsage { + // the node choice for this monitor + var nodeChoice *NodeUsage + + // for each zone, in order of preference. unlabeled nodes are last, by + // construction; see Cluster.getNodeMonusage(). + for zi := range nodeZones { + // number of monitors in the zone + zoneMonCount := 0 + + // preferential node choice from this zone for a new mon + var zoneNodeChoice *NodeUsage + + // for each node in this zone + for ni := range nodeZones[zi] { + nodeUsage := &nodeZones[zi][ni] + + // skip invalid nodes. but update the mon count first since + // invalid nodes could still have an assigned mon pod. + zoneMonCount += nodeUsage.MonCount + if !nodeUsage.MonValid { + logger.Infof("schedmon: skip invalid node %s for mon scheduling", + nodeUsage.Node.Name) + continue + } + + // make a "best" choice from this zone. in this case that is the + // node with the least amount of monitors. + if zoneNodeChoice == nil || nodeUsage.MonCount < zoneNodeChoice.MonCount { + logger.Infof("schedmon: considering node %s with mon count %d", + nodeUsage.Node.Name, nodeUsage.MonCount) + zoneNodeChoice = nodeUsage + } + } + + // We identified _a_ valid node in the zone. should we choose it? + if zoneNodeChoice != nil { + if zoneMonCount == 0 { + // this zone has no monitors, which implies that + // zoneNodeChoice will be a node without any monitors + // currently assigned. choose this node. + logger.Infof("schedmon: considering node %s from empty zone", + zoneNodeChoice.Node.Name) + nodeChoice = zoneNodeChoice + break + } else { + // this zone already has a monitor in it. but keep the + // choice as a backup; we may find that all zones already + // have a monitor and still need to make an assignment. + if nodeChoice == nil || zoneNodeChoice.MonCount < nodeChoice.MonCount { + logger.Infof("schedmon: considering node %s with mon count %d", + zoneNodeChoice.Node.Name, zoneNodeChoice.MonCount) + nodeChoice = zoneNodeChoice + } + } + } } - nodeIndex := 0 - for _, m := range mons { - if _, ok := c.mapping.Node[m.DaemonName]; ok { - logger.Debugf("mon %s already assigned to a node, no need to assign", m.DaemonName) + if nodeChoice != nil { + logger.Infof("schedmon: scheduling mon %s on node %s", + mon.DaemonName, nodeChoice.Node.Name) + } else { + logger.Infof("schedmon: no suitable node found for mon %s", + mon.DaemonName) + } + + return nodeChoice +} + +func (c *Cluster) assignMons(mons []*monConfig) error { + + // retrieve the set of cluster nodes and their monitor usage info + nodeZones, err := c.getNodeMonUsage() + if err != nil { + return fmt.Errorf("failed to get node monitor usage. %+v", err) + } + + // ensure all monitors have a node assignment. note that this isn't + // necessarily optimal: it does not try to move existing monitors which is + // handled by the periodic monitor health checks. + for _, mon := range mons { + + // monitor is already assigned to a node. nothing to do + if _, ok := c.mapping.Node[mon.DaemonName]; ok { + logger.Infof("assignmon: mon %s already assigned to a node", mon.DaemonName) continue } - // if we need to place a new mon and don't have any more nodes available, we fail to add the mon - if len(availableNodes) == 0 { - return fmt.Errorf("no nodes available for mon placement") + nodeChoice := scheduleMonitor(mon, nodeZones) + + if nodeChoice == nil { + return fmt.Errorf("assignmon: no valid nodes available for mon placement") } - // pick one of the available nodes where the mon will be assigned - node := availableNodes[nodeIndex%len(availableNodes)] - logger.Debugf("mon %s assigned to node %s", m.DaemonName, node.Name) - nodeInfo, err := getNodeInfoFromNode(node) - if err != nil { - return fmt.Errorf("couldn't get node info from node %s. %+v", node.Name, err) + // note that we do not need to worry about a false negative here (i.e. a + // node exists that _would_ pass this check) due to the search landing in a + // local minima because if a node had existed with a mon count of zero, + // scheduleMonitor would have chosen it over any node with a positive + // mon count. + if nodeChoice.MonCount > 0 && !c.spec.Mon.AllowMultiplePerNode { + return fmt.Errorf("assignmon: no empty nodes available for mon placement") } - c.mapping.Node[m.DaemonName] = nodeInfo - nodeIndex++ + + // make this decision visible when scheduling the next monitor + nodeChoice.MonCount++ + + logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Node.Name) + + nodeInfo, err := getNodeInfoFromNode(*nodeChoice.Node) + if err != nil { + return fmt.Errorf("couldn't get node info from node %s. %+v", + nodeChoice.Node.Name, err) + } + + c.mapping.Node[mon.DaemonName] = nodeInfo } - logger.Debug("mons have been assigned to nodes") + logger.Debug("assignmons: mons have been assigned to nodes") return nil } diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 8307b4b92..fc9a94bf2 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -339,3 +339,168 @@ func TestMonFoundInQuorum(t *testing.T) { assert.True(t, monFoundInQuorum("c", response)) assert.False(t, monFoundInQuorum("d", response)) } + +// no node choice can be made when there are no nodes +func TestScheduleMonitorEmpty(t *testing.T) { + nodeZones := [][]NodeUsage{} + mon := &monConfig{DaemonName: "a"} + // no zones + assert.Nil(t, scheduleMonitor(mon, nodeZones)) + // 1 zone no mons + nodeZones = append(nodeZones, []NodeUsage{}) + assert.Nil(t, scheduleMonitor(mon, nodeZones)) + // 2 zones no mons + nodeZones = append(nodeZones, []NodeUsage{}) + assert.Nil(t, scheduleMonitor(mon, nodeZones)) +} + +// only valid nodes should be chosen +func TestScheduleMonitorInvalidNodes(t *testing.T) { + // 1 zone with 1 invalid empty node + nodeZones := [][]NodeUsage{ + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, + }, + } + mon := &monConfig{DaemonName: "a"} + assert.Nil(t, scheduleMonitor(mon, nodeZones)) + + // 1 zone with 2 invalid empty node + nodeZones = [][]NodeUsage{ + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, + }, + } + assert.Nil(t, scheduleMonitor(mon, nodeZones)) + + // 2 zone with 2 invalid empty node each + nodeZones = [][]NodeUsage{ + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, + }, + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, + }, + } + assert.Nil(t, scheduleMonitor(mon, nodeZones)) +} + +func TestScheduleMonitor(t *testing.T) { + // 1 zone, 1 valid, empty node -> only one choice + nodeZones := [][]NodeUsage{ + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + }, + } + mon := &monConfig{DaemonName: "a"} + assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) + + // still the only choice even if not empty + nodeZones = [][]NodeUsage{ + { + NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, + }, + } + assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) + + // scheduler prefers the node with the least number of mons + nodeZones = [][]NodeUsage{ + { + NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, + }, + } + assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones)) + + // the scheduler prefers nodes with the least number of mons, not zones with + // the leads number of mons (zero mons is a special case: tested below...) + nodeZones = [][]NodeUsage{ + // 24 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 4, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, + }, + // 10 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, + }, + } + // choose the node with 4 mons + assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones)) + + // same as before, the target mon is in the second zone + nodeZones = [][]NodeUsage{ + // 6 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, + }, + // 10 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 9, MonValid: true}, + }, + } + // choose the node with 1 mon + assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) + + // prefers a zone with zero mons to spread across failure domains + nodeZones = [][]NodeUsage{ + // 0 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + }, + // 1 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, + }, + } + // choose the zone with zero mons + assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) + + // same as before, different zone + nodeZones = [][]NodeUsage{ + // 0 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + }, + // 1 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + }, + } + // choose the zone with zero mons + assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) + + // invalid nodes aren't schedulable, but if they have mons, that factors + // into the decision. in this case it means the zone isn't really empty + nodeZones = [][]NodeUsage{ + // 0 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + // invalid node, but has a mon -> zone is not empty + NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: false}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + }, + // 1 mons + { + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, + }, + } + // choose the zone with zero mons + assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) +} diff --git a/pkg/operator/ceph/cluster/mon/node.go b/pkg/operator/ceph/cluster/mon/node.go index 1de28622a..624df5d27 100644 --- a/pkg/operator/ceph/cluster/mon/node.go +++ b/pkg/operator/ceph/cluster/mon/node.go @@ -21,85 +21,19 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/k8sutil" - "github.com/rook/rook/pkg/util" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// getMonNodes detects the nodes that are available for new mons to start. -func (c *Cluster) getMonNodes() ([]v1.Node, error) { - availableNodes, nodes, err := c.getAvailableMonNodes() - if err != nil { - return nil, err - } - logger.Infof("Found %d running nodes without mons", len(availableNodes)) - - // if all nodes already have mons and the user has given the mon.count, add all nodes to be available - if c.spec.Mon.AllowMultiplePerNode && len(availableNodes) == 0 { - logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items)) - for _, node := range nodes.Items { - valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - } else if valid { - availableNodes = append(availableNodes, node) - } - } - } - - return availableNodes, nil -} - -func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) { - nodeOptions := metav1.ListOptions{} - nodeOptions.TypeMeta.Kind = "Node" - nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions) - if err != nil { - return nil, nil, err - } - logger.Debugf("there are %d nodes available for %d mons", len(nodes.Items), len(c.clusterInfo.Monitors)) - - // get the nodes that have mons assigned - nodesInUse, err := c.getNodesWithMons(nodes) - if err != nil { - logger.Warningf("could not get nodes with mons. %+v", err) - nodesInUse = util.NewSet() - } - - // choose nodes for the new mons that don't have mons currently - availableNodes := []v1.Node{} - for _, node := range nodes.Items { - if !nodesInUse.Contains(node.Name) { - valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - } else if valid { - availableNodes = append(availableNodes, node) - } - } - } - - return availableNodes, nodes, nil -} - -func (c *Cluster) getNodesWithMons(nodes *v1.NodeList) (*util.Set, error) { - // get the mon pods and their node affinity - options := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", appName)} - pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(options) - if err != nil { - return nil, err - } - nodesInUse := util.NewSet() - for _, pod := range pods.Items { - hostname := pod.Spec.NodeSelector[v1.LabelHostname] - logger.Debugf("mon pod on node %s", hostname) - name, ok := getNodeNameFromHostname(nodes, hostname) - if !ok { - logger.Errorf("mon %s on hostname %s not found in node list", pod.Name, hostname) - } - nodesInUse.Add(name) - } - return nodesInUse, nil +// NodeUsage is a mapping between a Node and computed metadata about the node +// that is used in monitor pod scheduling. +type NodeUsage struct { + Node *v1.Node + // The number of monitor pods assigned to the node + MonCount int + // The node is available for scheduling monitor pods. This is equivalent to + // evaluating k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) + MonValid bool } // Get the number of mons that the operator should be starting @@ -190,3 +124,76 @@ func getNodeInfoFromNode(n v1.Node) (*NodeInfo, error) { } return nr, nil } + +// Returns a hierarchical representation of the nodes in the cluster organized +// by zone failure domain and annotated with the number of monitor pods assigned +// to each node and if the node is valid for scheduling monitor pods. The +// purpose of this structure is to provide a unified view of information +// required for making monitor scheduling decisions. +func (c *Cluster) getNodeMonUsage() ([][]NodeUsage, error) { + // get all k8s node objects + nodeOptions := metav1.ListOptions{} + nodeOptions.TypeMeta.Kind = "Node" + nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions) + if err != nil { + return nil, err + } + + // get all pod objects labeled as a monitor + podOptions := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", appName)} + pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(podOptions) + if err != nil { + return nil, err + } + + // generate a list of nodes that includes the number of monitor pods + // assigned to each node. this is equivalent to: + // + // SELECT Node, Count(*) FROM Nodes + // JOIN Pods ON Nodes.Hostname == Pods.Hostname + // GROUP BY Node + // + nodeUsages := []NodeUsage{} + for i, node := range nodes.Items { + valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) + if err != nil { + logger.Warning("failed to validate node %s %v", node.Name, err) + continue + } + nodeUsage := NodeUsage{Node: &nodes.Items[i], MonCount: 0, MonValid: valid} + for _, pod := range pods.Items { + hostname := pod.Spec.NodeSelector[v1.LabelHostname] + if node.Name == hostname || node.Labels[v1.LabelHostname] == hostname { + nodeUsage.MonCount++ + } + } + nodeUsages = append(nodeUsages, nodeUsage) + } + + // generate a hierarchical representation of the nodes grouped by the name + // of the node's zone failure domain label. the zone label named "" is used + // for nodes without a zone annotation. + nodesByZone := make(map[string][]NodeUsage) + for _, nodeUsage := range nodeUsages { + zone, ok := nodeUsage.Node.Labels["failure-domain.beta.kubernetes.io/zone"] + if !ok { + zone = "" + } + nodesByZone[zone] = append(nodesByZone[zone], nodeUsage) + } + + // compute the final form used for scheduling. this form is an array of + // nodes by zone, with non-labeled nodes appearing in the final zone. this + // organization reflects the preference made by the scheduler of using nodes + // with zone annotations before non-labeled nodes. + res := [][]NodeUsage{} + for zone, nodeUsage := range nodesByZone { + if zone == "" { + res = append(res, nodeUsage) + } else { + res = append([][]NodeUsage{nodeUsage}, res...) + } + } + + return res, nil +} diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index 601786fdc..833bfa282 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -31,114 +31,115 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestAvailableMonNodes(t *testing.T) { - clientset := test.New(1) +func TestGetNodeMonUsageValidNode(t *testing.T) { + clientset := test.New(2) c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{}) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - nodes, err := c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 1, len(nodes)) - - // set node to not ready + node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) conditions := v1.NodeCondition{Type: v1.NodeOutOfDisk} - nodes[0].Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}} - clientset.CoreV1().Nodes().Update(&nodes[0]) + node.Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}} + clientset.CoreV1().Nodes().Update(node) - // when the node is not ready there should be no nodes returned and an error - emptyNodes, err := c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 0, len(emptyNodes)) - - // even if AllowMultiplePerNode is true there should be no node returned - c.spec.Mon.AllowMultiplePerNode = true - emptyNodes, err = c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 0, len(emptyNodes)) + // nodes are reported with correct valid flag + nodeZones, err := c.getNodeMonUsage() + assert.NoError(t, err) + assert.False(t, nodeZones[0][0].MonValid) + assert.True(t, nodeZones[0][1].MonValid) } -func TestAvailableNodesInUse(t *testing.T) { - clientset := test.New(3) +func TestGetNodeMonUsageMonCount(t *testing.T) { + clientset := test.New(2) c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{}) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - // all three nodes are available by default - nodes, err := c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 3, len(nodes)) + // 3 mons on node0 + node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) - // start pods on two of the nodes so that only one node will be available - monIDs := []string{"a", "b"} - for i := 0; i < 2; i++ { - pod := c.makeMonPod(testGenMonConfig(monIDs[i]), nodes[i].Name) + monIDs := []string{"a", "b", "c"} + for i := 0; i < 3; i++ { + pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name) _, err = clientset.CoreV1().Pods(c.Namespace).Create(pod) assert.Nil(t, err) } - reducedNodes, err := c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 1, len(reducedNodes)) - assert.Equal(t, nodes[2].Name, reducedNodes[0].Name) - // start pods on the remaining node. We expect no nodes to be available for placement - // since there is no way to place a mon on an unused node. - pod := c.makeMonPod(testGenMonConfig("c"), nodes[2].Name) - _, err = clientset.CoreV1().Pods(c.Namespace).Create(pod) - assert.Nil(t, err) - nodes, err = c.getMonNodes() - // no mon nodes is no error, just an empty nodes list - assert.Nil(t, err) - assert.Equal(t, 3, len(nodes)) + // 2 mons on node1 + node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) - // no nodes should be returned when AllowMultiplePerNode is false - c.spec.Mon.AllowMultiplePerNode = false - nodes, err = c.getMonNodes() - // no mon nodes is no error, just an empty nodes list - assert.Nil(t, err) - assert.Equal(t, 0, len(nodes)) + monIDs = []string{"d", "e"} + for i := 0; i < 2; i++ { + pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name) + _, err = clientset.CoreV1().Pods(c.Namespace).Create(pod) + assert.Nil(t, err) + } + + nodeZones, err := c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, 1, len(nodeZones)) + if nodeZones[0][0].Node.Name == "node0" { + assert.Equal(t, 3, nodeZones[0][0].MonCount) + assert.Equal(t, 2, nodeZones[0][1].MonCount) + } else { + assert.Equal(t, 2, nodeZones[0][1].MonCount) + assert.Equal(t, 3, nodeZones[0][0].MonCount) + } } func TestTaintedNodes(t *testing.T) { - clientset := test.New(3) + clientset := test.New(4) c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{}) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - nodes, err := c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 3, len(nodes)) - // mark a node as unschedulable - nodes[0].Spec.Unschedulable = true - clientset.CoreV1().Nodes().Update(&nodes[0]) - nodesSchedulable, err := c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 2, len(nodesSchedulable)) - nodes[0].Spec.Unschedulable = false + node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + node.Spec.Unschedulable = true + clientset.CoreV1().Nodes().Update(node) - // taint nodes so they will not be schedulable for new pods - nodes[0].Spec.Taints = []v1.Taint{ + // taint + node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + node.Spec.Taints = []v1.Taint{ {Effect: v1.TaintEffectNoSchedule}, } - nodes[1].Spec.Taints = []v1.Taint{ + clientset.CoreV1().Nodes().Update(node) + + // taint + node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + node.Spec.Taints = []v1.Taint{ {Effect: v1.TaintEffectPreferNoSchedule}, } - clientset.CoreV1().Nodes().Update(&nodes[0]) - clientset.CoreV1().Nodes().Update(&nodes[1]) - clientset.CoreV1().Nodes().Update(&nodes[2]) - cleanNodes, err := c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 1, len(cleanNodes)) - assert.Equal(t, nodes[2].Name, cleanNodes[0].Name) + clientset.CoreV1().Nodes().Update(node) + + nodeZones, err := c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, 1, len(nodeZones)) + validCount := 0 + for zi := range nodeZones { + for ni := range nodeZones[zi] { + if nodeZones[zi][ni].MonValid { + validCount++ + } + } + } + assert.Equal(t, 1, validCount) } func TestNodeAffinity(t *testing.T) { - clientset := test.New(3) + clientset := test.New(4) c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{}) setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - nodes, err := c.getMonNodes() - assert.Nil(t, err) - assert.Equal(t, 3, len(nodes)) - c.spec.Placement = map[rookalpha.KeyType]rookalpha.Placement{} c.spec.Placement[cephv1.KeyMon] = rookalpha.Placement{NodeAffinity: &v1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ @@ -157,19 +158,31 @@ func TestNodeAffinity(t *testing.T) { }, } - // label nodes so node0 will not be schedulable for new pods - nodes[0].Labels = map[string]string{"label": "foo"} - nodes[1].Labels = map[string]string{"label": "bar"} - nodes[2].Labels = map[string]string{"label": "baz"} - clientset.CoreV1().Nodes().Update(&nodes[0]) - clientset.CoreV1().Nodes().Update(&nodes[1]) - clientset.CoreV1().Nodes().Update(&nodes[2]) - cleanNodes, err := c.getMonNodes() + // label nodes so they appear as not scheduable / invalid + node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + node.Labels = map[string]string{"label": "foo"} + clientset.CoreV1().Nodes().Update(node) - assert.Nil(t, err) - assert.Equal(t, 2, len(cleanNodes)) - assert.Equal(t, nodes[1].Name, cleanNodes[0].Name) - assert.Equal(t, nodes[2].Name, cleanNodes[1].Name) + node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) + node.Labels = map[string]string{"label": "bar"} + clientset.CoreV1().Nodes().Update(node) + + node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) + node.Labels = map[string]string{"label": "baz"} + clientset.CoreV1().Nodes().Update(node) + + nodeZones, err := c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, 1, len(nodeZones)) + validCount := 0 + for zi := range nodeZones { + for ni := range nodeZones[zi] { + if nodeZones[zi][ni].MonValid { + validCount++ + } + } + } + assert.Equal(t, 2, validCount) } // this tests can 3 mons with hostnetworking on the same host is rejected @@ -268,8 +281,18 @@ func TestHostNetwork(t *testing.T) { c.HostNetwork = true - nodes, err := c.getMonNodes() + nodes := []v1.Node{} + nodeZones, err := c.getNodeMonUsage() assert.Nil(t, err) + + for zi := range nodeZones { + for _, nodeUsage := range nodeZones[zi] { + if nodeUsage.MonCount == 0 || c.spec.Mon.AllowMultiplePerNode { + nodes = append(nodes, *nodeUsage.Node) + } + } + } + assert.Equal(t, 3, len(nodes)) monConfig := testGenMonConfig("c") @@ -375,3 +398,134 @@ func TestTargetMonCount(t *testing.T) { assert.Equal(t, 7, target) logger.Infof(msg) } + +// mon node usage should return no zones if there are no nodes +func TestGetNodeMonUsageNoNodes(t *testing.T) { + clientset := test.New(0) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{}) + setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") + + nodeZones, err := c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, len(nodeZones), 0) +} + +// all nodes without zone annotations are in the same zone +func TestGetNodeMonUsageNoZoneLabels(t *testing.T) { + for i := 1; i < 5; i++ { + clientset := test.New(i) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{}) + setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") + + nodeZones, err := c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, len(nodeZones), 1) + assert.Equal(t, len(nodeZones[0]), i) + } +} + +// nodes are partitioned into separate zones +func TestGetNodeMonUsageZoneSpread(t *testing.T) { + clientset := test.New(5) + c := New(&clusterd.Context{Clientset: clientset}, "ns", "", false, metav1.OwnerReference{}) + setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") + + // 1 node labeled -> 1 zone + the rest in the unlabeled zone + node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"} + clientset.CoreV1().Nodes().Update(node) + + // the zones are unlabeled and sort order isn't guaranteed. so instead we + // assert on expected count of zones and collection of zone sizes. + zoneCounts := map[int]int{} + + nodeZones, err := c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, 2, len(nodeZones)) + for _, zones := range nodeZones { + zoneSize := len(zones) + if _, ok := zoneCounts[zoneSize]; ok { + zoneCounts[zoneSize]++ + } else { + zoneCounts[zoneSize] = 1 + } + } + assert.Equal(t, map[int]int{1: 1, 4: 1}, zoneCounts) + + // 2 nodes with same label, 3 unlabeled + node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"} + clientset.CoreV1().Nodes().Update(node) + + // reset + zoneCounts = map[int]int{} + + nodeZones, err = c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, 2, len(nodeZones)) + for _, zones := range nodeZones { + zoneSize := len(zones) + if _, ok := zoneCounts[zoneSize]; ok { + zoneCounts[zoneSize]++ + } else { + zoneCounts[zoneSize] = 1 + } + } + assert.Equal(t, map[int]int{2: 1, 3: 1}, zoneCounts) + + // 2 nodes with same label, 2 with distinct labels, 1 unlabeled + node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"} + clientset.CoreV1().Nodes().Update(node) + + node, err = clientset.CoreV1().Nodes().Get("node3", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z2"} + clientset.CoreV1().Nodes().Update(node) + + // reset + zoneCounts = map[int]int{} + + nodeZones, err = c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, 4, len(nodeZones)) + for _, zones := range nodeZones { + zoneSize := len(zones) + if _, ok := zoneCounts[zoneSize]; ok { + zoneCounts[zoneSize]++ + } else { + zoneCounts[zoneSize] = 1 + } + } + assert.Equal(t, map[int]int{2: 1, 1: 3}, zoneCounts) + + // no unlabeled zones + node, err = clientset.CoreV1().Nodes().Get("node4", metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, node) + node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"} + clientset.CoreV1().Nodes().Update(node) + + // reset + zoneCounts = map[int]int{} + + nodeZones, err = c.getNodeMonUsage() + assert.NoError(t, err) + assert.Equal(t, 3, len(nodeZones)) + for _, zones := range nodeZones { + zoneSize := len(zones) + if _, ok := zoneCounts[zoneSize]; ok { + zoneCounts[zoneSize]++ + } else { + zoneCounts[zoneSize] = 1 + } + } + assert.Equal(t, map[int]int{2: 2, 1: 1}, zoneCounts) +} diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go index ad3fb250a..1102be478 100644 --- a/pkg/operator/k8sutil/node.go +++ b/pkg/operator/k8sutil/node.go @@ -37,7 +37,8 @@ var validTopologyLabelKeys = []string{ } // ValidNodeNoSched returns true if the node (1) meets Rook's placement terms, -// and (2) is ready. False otherwise. +// and (2) is ready. Unlike ValidNode, this method will ignore the +// Node.Spec.Unschedulable flag. False otherwise. func ValidNodeNoSched(node v1.Node, placement rookalpha.Placement) (bool, error) { p, err := NodeMeetsPlacementTerms(node, placement, false) if err != nil {