Merge pull request #3668 from noahdesu/canary-sched
ceph/mon: support monitor migration without rebuils
This commit is contained in:
18 files changed
+455
-1385
No files matched your search
@@ -130,10 +130,6 @@ For more details on the mons and when to choose a number other than `3`, see the
|
||||
### Mon Settings
|
||||
|
||||
- `count`: Set the number of mons to be started. The number should be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`.
|
||||
- `preferredCount`: If you want to increase the number of mons when the number of nodes increases, set the `preferredCount` to be larger than the `count`. For example, if your cluster
|
||||
starts with three nodes, but might grow to more than five nodes, you might want five mons running after the other nodes come online. In this case, set `count: 3` and `preferredCount: 5`.
|
||||
When the operator sees the new nodes come online, the number of mons will increase to the preferred count. If the number of nodes decreases below the `preferredCount`, the operator will
|
||||
reduce the number of mons back to `count`. If `allowMultiplePerNode: true` (for testing scenarios), the number of mons will always use `preferredCount` if set.
|
||||
- `allowMultiplePerNode`: Enable (`true`) or disable (`false`) the placement of multiple mons on one node. Default is `false`.
|
||||
- `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs
|
||||
for monitor storage. This field is optional, and when not provided, HostPath
|
||||
|
||||
@@ -59,6 +59,7 @@ an example usage
|
||||
- The minimum version supported by Rook is Ceph Mimic v13.2.4. Before upgrading to v1.1 it is required to update the version of Ceph to at least this version.
|
||||
- The CSI driver is enabled by default. Documentation has been changed significantly for block and filesystem to use the CSI driver instead of flex.
|
||||
While the flex driver is still supported, it is anticipated to be deprecated soon.
|
||||
- The `Mon.PreferredCount` setting has been removed.
|
||||
|
||||
## Known Issues
|
||||
|
||||
|
||||
@@ -47,10 +47,6 @@ spec:
|
||||
maximum: 9
|
||||
minimum: 1
|
||||
type: integer
|
||||
preferredCount:
|
||||
maximum: 9
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- count
|
||||
network:
|
||||
|
||||
@@ -79,6 +79,10 @@ spec:
|
||||
# operator: Exists
|
||||
# The above placement information can also be specified for mon, osd, and mgr components
|
||||
# mon:
|
||||
# Monitor deployments may contain an anti-affinity rule for avoiding monitor
|
||||
# colocation on the same node. This is a required rule when host network is used
|
||||
# or when AllowMultiplePerNode is false. Otherwise this anti-affinity rule is a
|
||||
# preferred rule with weight: 50.
|
||||
# osd:
|
||||
# mgr:
|
||||
annotations:
|
||||
|
||||
@@ -66,10 +66,6 @@ spec:
|
||||
maximum: 9
|
||||
minimum: 0
|
||||
type: integer
|
||||
preferredCount:
|
||||
maximum: 9
|
||||
minimum: 0
|
||||
type: integer
|
||||
network:
|
||||
properties:
|
||||
hostNetwork:
|
||||
|
||||
+47
-6
@@ -23,7 +23,6 @@ claim template that is used to generate PVCs for monitor database storage.
|
||||
```go
|
||||
type MonSpec struct {
|
||||
Count int `json:"count"`
|
||||
PreferredCount int `json:"preferredCount"`
|
||||
AllowMultiplePerNode bool `json:"allowMultiplePerNode"`
|
||||
VolumeClaimTemplate *v1.PersistentVolumeClaim
|
||||
}
|
||||
@@ -89,9 +88,51 @@ scheduling implementation in Rook. These restrictions will be removed or
|
||||
significantly relaxed once monitor scheduling is moved under control of Kubernetes
|
||||
itself (ongoing work).
|
||||
|
||||
## Migration
|
||||
## Scheduling
|
||||
|
||||
Monitor migration is currently not implemented. When nodes are decommissioned,
|
||||
for example, any monitors on the nodes are migrated using a fail over mechanism
|
||||
rather than also migrating the PVC. This support is currently being worked on in
|
||||
conjunction with the enhancements to scheduling.
|
||||
In previous versions of Rook the operator made explicit scheduling (placement)
|
||||
decisions when creating monitor deployments. These decisions were made by
|
||||
implementing a custom scheduling algorithm, and using the pod node selector to
|
||||
enforce the placement decision. Unfortunately, schedulers are difficult to
|
||||
write correctly, and manage. Furthermore, by maintaining a separate scheduler
|
||||
from Kubernetes global policies are difficult to achieve.
|
||||
|
||||
Despite the benefits of using the Kubernetes scheduler, there are important use
|
||||
cases for using a node selector on a monitor deployment: pinning a monitor to a
|
||||
node when HostPath-based storage is used. In this case Rook must prevent k8s
|
||||
from moving a pod away from the node that contains its storage. The node
|
||||
selector is used to enforce this affinity. Unfortunately, node selector use is
|
||||
mutually exclusive with kubernetes scheduling---a pod cannot be scheduled by
|
||||
Kubernetes and then atomically have its affinity set to that placement decision.
|
||||
|
||||
The workaround in Rook is to use a temporary _canary pod_ that is scheduled by
|
||||
Kubernetes, but whose placement is enforced by Rook. The canary deployment is a
|
||||
deployment configured identically to a monitor deployment, except the container
|
||||
entrypoints have no side affects. The canary deployments are used to solve a
|
||||
fundamental bootstrapping issue: we want to avoid making explicit scheduling
|
||||
decisions in Rook, but in some configurations a node selector needs to be used
|
||||
to pin a monitor to a node.
|
||||
|
||||
### Health checks
|
||||
|
||||
Previous versions of Rook performed periodic health checks that included checks
|
||||
on monitor health as well as looking for scheduling violations. The health
|
||||
checks related to scheduling violations have been removed. Fundamentally a
|
||||
placement violation requires understanding or accessing the scheduling algorithm.
|
||||
|
||||
The rescheduling or eviction aspects of Rook's scheduling caused more problems
|
||||
than it helped, so going with K8s scheduling is the right thing. If/when K8s
|
||||
has eviction policies in the future we could then make use of it (e.g. with
|
||||
`*RequiredDuringExecution` variants of anti-affinity rules are available).
|
||||
|
||||
### Target Monitor Count
|
||||
|
||||
The `PreferredCount` feature has been removed.
|
||||
|
||||
The CRD monitor count specifies a target minimum number of monitors to maintain.
|
||||
Additionally, a preferred count is available which will be the desired number of
|
||||
sufficient number of nodes are available. Unfortunately, this calculation
|
||||
relies on determining if monitor pods may be placed on a node, requiring
|
||||
knowledge of the scheduling policy and algorithm. The scenario to watch out for
|
||||
is an endless loop in which the health check is determining a bad placement but
|
||||
the k8s schedule thinks otherwise.
|
||||
@@ -150,7 +150,6 @@ const (
|
||||
|
||||
type MonSpec struct {
|
||||
Count int `json:"count,omitempty"`
|
||||
PreferredCount int `json:"preferredCount,omitempty"`
|
||||
AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"`
|
||||
VolumeClaimTemplate *v1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"`
|
||||
}
|
||||
|
||||
@@ -81,7 +81,6 @@ func CreateOrLoadClusterInfo(context *clusterd.Context, namespace string, ownerR
|
||||
maxMonID := -1
|
||||
monMapping := &Mapping{
|
||||
Node: map[string]*NodeInfo{},
|
||||
Port: map[string]int32{},
|
||||
}
|
||||
|
||||
secrets, err := context.Clientset.CoreV1().Secrets(namespace).Get(AppName, metav1.GetOptions{})
|
||||
@@ -137,7 +136,6 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string
|
||||
maxMonID := -1
|
||||
monMapping := &Mapping{
|
||||
Node: map[string]*NodeInfo{},
|
||||
Port: map[string]int32{},
|
||||
}
|
||||
|
||||
cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(EndpointConfigMapName, metav1.GetOptions{})
|
||||
|
||||
@@ -25,11 +25,8 @@ import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
cephutil "github.com/rook/rook/pkg/daemon/ceph/util"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
scheduler "k8s.io/kubernetes/pkg/scheduler/api"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -95,11 +92,8 @@ func (c *Cluster) checkHealth() error {
|
||||
|
||||
// Use a local mon count in case the user updates the crd in another goroutine.
|
||||
// We need to complete a health check with a consistent value.
|
||||
desiredMonCount, msg, err := c.getTargetMonCount()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get target mon count. %+v", err)
|
||||
}
|
||||
logger.Debugf(msg)
|
||||
desiredMonCount := c.spec.Mon.Count
|
||||
logger.Debugf("targeting the mon count %d", desiredMonCount)
|
||||
|
||||
// Source of truth of which mons should exist is our *clusterInfo*
|
||||
monsNotFound := map[string]interface{}{}
|
||||
@@ -172,13 +166,6 @@ func (c *Cluster) checkHealth() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// find any mons that invalidate our placement policy, and if necessary,
|
||||
// reschedule them to other nodes.
|
||||
done, err := c.resolveInvalidMonitorPlacement(desiredMonCount)
|
||||
if done || err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create/start new mons when there are fewer mons than the desired count in the CRD
|
||||
if len(status.MonMap.Mons) < desiredMonCount {
|
||||
logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(status.MonMap.Mons), desiredMonCount)
|
||||
@@ -198,54 +185,6 @@ func (c *Cluster) checkHealth() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) checkMonsOnSameNode(desiredMonCount int) (bool, error) {
|
||||
nodesUsed := map[string]struct{}{}
|
||||
for name, node := range c.mapping.Node {
|
||||
// when the node is already in the list we have more than one mon on that node
|
||||
if _, ok := nodesUsed[node.Name]; ok {
|
||||
// get list of available nodes for mons
|
||||
availableNodes, _, err := c.getAvailableMonNodes()
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("failed to get available mon nodes. %+v", err)
|
||||
}
|
||||
// if there are enough nodes for one mon "that is too much" to be failovered,
|
||||
// fail it over to an other node
|
||||
if len(availableNodes) > 0 {
|
||||
logger.Infof("rebalance: enough nodes available %d to failover mon %s", len(availableNodes), name)
|
||||
c.failMon(len(c.ClusterInfo.Monitors), desiredMonCount, name)
|
||||
} else {
|
||||
logger.Debugf("rebalance: not enough nodes available to failover mon %s", name)
|
||||
}
|
||||
|
||||
// deal with one mon too much on a node at a time
|
||||
return true, nil
|
||||
}
|
||||
nodesUsed[node.Name] = struct{}{}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) checkMonsOnValidNodes() (bool, error) {
|
||||
for mon, nInfo := range c.mapping.Node {
|
||||
// get node to use for validNode() func
|
||||
node, err := c.context.Clientset.CoreV1().Nodes().Get(nInfo.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
// check if node the mon is on is still valid
|
||||
valid, err := k8sutil.ValidNode(*node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if !valid {
|
||||
logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon)
|
||||
c.failoverMon(mon)
|
||||
return true, nil
|
||||
}
|
||||
logger.Debugf("node %s with mon %s is still valid", nInfo.Name, mon)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// failMon compares the monCount against desiredMonCount
|
||||
func (c *Cluster) failMon(monCount, desiredMonCount int, name string) {
|
||||
if monCount > desiredMonCount {
|
||||
@@ -268,16 +207,10 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
m := c.newMonConfig(c.maxMonID + 1)
|
||||
logger.Infof("starting new mon: %+v", m)
|
||||
|
||||
// Create the service endpoint
|
||||
serviceIP, err := c.createService(m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create mon service. %+v", err)
|
||||
}
|
||||
|
||||
mConf := []*monConfig{m}
|
||||
|
||||
// Assign the pod to a node
|
||||
if err = c.assignMons(mConf); err != nil {
|
||||
if err := c.assignMons(mConf); err != nil {
|
||||
return fmt.Errorf("failed to place new mon on a node. %+v", err)
|
||||
}
|
||||
|
||||
@@ -288,12 +221,17 @@ func (c *Cluster) failoverMon(name string) error {
|
||||
}
|
||||
m.PublicIP = node.Address
|
||||
} else {
|
||||
// Create the service endpoint
|
||||
serviceIP, err := c.createService(m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create mon service. %+v", err)
|
||||
}
|
||||
m.PublicIP = serviceIP
|
||||
}
|
||||
c.ClusterInfo.Monitors[m.DaemonName] = cephconfig.NewMonInfo(m.DaemonName, m.PublicIP, m.Port)
|
||||
|
||||
// Start the deployment
|
||||
if err = c.startDeployments(mConf, true); err != nil {
|
||||
if err := c.startDeployments(mConf, true); err != nil {
|
||||
return fmt.Errorf("failed to start new mon %s. %+v", m.DaemonName, err)
|
||||
}
|
||||
|
||||
@@ -327,19 +265,7 @@ func (c *Cluster) removeMon(daemonName string) error {
|
||||
delete(c.ClusterInfo.Monitors, daemonName)
|
||||
// check if a mapping exists for the mon
|
||||
if _, ok := c.mapping.Node[daemonName]; ok {
|
||||
nodeName := c.mapping.Node[daemonName].Name
|
||||
delete(c.mapping.Node, daemonName)
|
||||
// if node->port "mapping" has been created, decrease or delete it
|
||||
if port, ok := c.mapping.Port[nodeName]; ok {
|
||||
if port == DefaultMsgr1Port {
|
||||
delete(c.mapping.Port, nodeName)
|
||||
}
|
||||
// don't clean up if a node port is higher than the default port, other
|
||||
// mons could be on the same node with > DefaultPort ports, decreasing could
|
||||
// cause port collisions
|
||||
// This can be solved by using a map[nodeName][]int32 for the ports to
|
||||
// even better check which ports are open for the HostNetwork mode
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the service endpoint
|
||||
@@ -369,151 +295,6 @@ func removeMonitorFromQuorum(context *clusterd.Context, clusterName, name string
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) resolveInvalidMonitorPlacement(desiredMonCount int) (bool, error) {
|
||||
nodeChoice, err := c.findInvalidMonitorPlacement(desiredMonCount)
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("failed to find invalid mon placement %+v", err)
|
||||
}
|
||||
|
||||
// no violation found
|
||||
if nodeChoice == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for name, nodeInfo := range c.mapping.Node {
|
||||
if nodeInfo.Name == nodeChoice.Node.Name {
|
||||
logger.Infof("rebalance: rescheduling mon %s from node %s", name, nodeInfo.Name)
|
||||
c.failMon(len(c.ClusterInfo.Monitors), desiredMonCount, name)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
logger.Warningf("rebalance: no mon pod found on node %s", nodeChoice.Node.Name)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) findInvalidMonitorPlacement(desiredMonCount int) (*NodeUsage, error) {
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get node monitor usage. %+v", err)
|
||||
}
|
||||
|
||||
// compute two helpful global flags:
|
||||
// - does an empty zone exist
|
||||
// - does an empty node exist
|
||||
emptyZone := false
|
||||
emptyNode := false
|
||||
for zi := range nodeZones {
|
||||
monFoundInZone := false
|
||||
for ni := range nodeZones[zi] {
|
||||
nodeUsage := &nodeZones[zi][ni]
|
||||
if nodeUsage.MonCount > 0 {
|
||||
monFoundInZone = true
|
||||
} else if nodeUsage.MonValid {
|
||||
// only consider valid nodes since this flag determines if a mon
|
||||
// may be scheduled on to a new node.
|
||||
emptyNode = true
|
||||
}
|
||||
}
|
||||
if !monFoundInZone {
|
||||
// emptyZone is used below to imply that an empty zone exists AND
|
||||
// that zone contains a valid node (i.e. a node that new pods can be
|
||||
// assigned). this check enforces that assumption by checking that
|
||||
// the zone contains at least one valid node.
|
||||
validNodeInZone := false
|
||||
for _, nodeUsage := range nodeZones[zi] {
|
||||
if nodeUsage.MonValid {
|
||||
validNodeInZone = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if validNodeInZone {
|
||||
emptyZone = true
|
||||
}
|
||||
logger.Debugf("rebalance: empty zone found. validNodeInZone: %t, emptyZone %t",
|
||||
validNodeInZone, emptyZone)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("rebalance: desired mon count: %d, empty zone found: %t, empty node found: %t",
|
||||
desiredMonCount, emptyZone, emptyNode)
|
||||
|
||||
for zi := range nodeZones {
|
||||
zoneMonCount := 0
|
||||
var nodeChoice *NodeUsage
|
||||
|
||||
// for each node consider two cases: too many monitors are running on a
|
||||
// node, or monitors are running on invalid nodes.
|
||||
for ni := range nodeZones[zi] {
|
||||
nodeUsage := &nodeZones[zi][ni]
|
||||
zoneMonCount += nodeUsage.MonCount
|
||||
|
||||
// if this node has too many monitors, and an underused node exists,
|
||||
// then consider moving the monitor. note that this check is
|
||||
// independent of the setting `AllowMultiplePerNode` since we also
|
||||
// want to avoid in general keeping multiple monitors on one node.
|
||||
if nodeUsage.MonCount > 1 && emptyNode {
|
||||
logger.Infof("rebalance: chose overloaded node %s with %d mons",
|
||||
nodeUsage.Node.Name, nodeUsage.MonCount)
|
||||
nodeChoice = nodeUsage
|
||||
break
|
||||
}
|
||||
|
||||
// check for mons on invalid nodes. but reschedule pod only when it
|
||||
// is invalid for reasons other than schedulability which should not
|
||||
// affect pods that are already scheduled.
|
||||
if nodeUsage.MonCount > 0 && !nodeUsage.MonValid {
|
||||
placement := cephv1.GetMonPlacement(c.spec.Placement)
|
||||
placement.Tolerations = append(placement.Tolerations,
|
||||
v1.Toleration{
|
||||
Key: scheduler.TaintNodeUnschedulable,
|
||||
Effect: "",
|
||||
Operator: v1.TolerationOpExists,
|
||||
})
|
||||
valid, err := k8sutil.ValidNodeNoSched(*nodeUsage.Node, placement)
|
||||
if err != nil {
|
||||
logger.Warningf("rebalance: failed to validate node %s %+v",
|
||||
nodeUsage.Node.Name, err)
|
||||
} else if !valid {
|
||||
logger.Infof("rebalance: chose invalid node %s",
|
||||
nodeUsage.Node.Name)
|
||||
nodeChoice = nodeUsage
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we didn't already select a node with a monitor for rescheduling,
|
||||
// then consider that a zone may be overloaded and can be rebalanced.
|
||||
// this case occurs if the zone has more than 1 monitor, and some other
|
||||
// zone exists without any monitors.
|
||||
if nodeChoice == nil && zoneMonCount > 1 && emptyZone {
|
||||
for ni := range nodeZones[zi] {
|
||||
nodeUsage := &nodeZones[zi][ni]
|
||||
if nodeUsage.MonCount > 0 {
|
||||
// select the node with the most monitors assigned
|
||||
if nodeChoice == nil || nodeUsage.MonCount > nodeChoice.MonCount {
|
||||
nodeChoice = nodeUsage
|
||||
}
|
||||
}
|
||||
}
|
||||
if nodeChoice != nil {
|
||||
logger.Infof("rebalance: chose node %s from overloaded zone",
|
||||
nodeChoice.Node.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if nodeChoice != nil {
|
||||
return nodeChoice, nil
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("rebalance: no mon placement violations or fixes available")
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) handleExternalMonStatus(status client.MonStatusResponse) error {
|
||||
|
||||
changed, err := c.addOrRemoveExternalMonitor(status)
|
||||
|
||||
@@ -17,7 +17,6 @@ limitations under the License.
|
||||
package mon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
@@ -34,7 +33,6 @@ import (
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
@@ -66,9 +64,14 @@ func TestCheckHealth(t *testing.T) {
|
||||
Name: "node0",
|
||||
Address: "",
|
||||
}
|
||||
c.mapping.Port["node0"] = DefaultMsgr1Port
|
||||
c.maxMonID = 4
|
||||
|
||||
// mock out the scheduler to return node0
|
||||
scheduleMonitor = func(c *Cluster, mon *monConfig) (SchedulingResult, error) {
|
||||
node, _ := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
return SchedulingResult{Node: node}, nil
|
||||
}
|
||||
|
||||
err := c.checkHealth()
|
||||
assert.Nil(t, err)
|
||||
logger.Infof("mons after checkHealth: %v", c.ClusterInfo.Monitors)
|
||||
@@ -117,7 +120,6 @@ func TestCheckHealthNotFound(t *testing.T) {
|
||||
c.mapping.Node["b"] = &NodeInfo{
|
||||
Name: "node0",
|
||||
}
|
||||
c.mapping.Port["node0"] = DefaultMsgr1Port
|
||||
c.maxMonID = 4
|
||||
|
||||
c.saveMonConfig()
|
||||
@@ -145,304 +147,6 @@ func TestCheckHealthNotFound(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverloadedZoneRebalances(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
|
||||
// build a cluster with 3 nodes. two of the nodes are in the same AZ the
|
||||
// third node is in a separate AZ.
|
||||
for i := 0; i < 3; i++ {
|
||||
nodeName := fmt.Sprintf("node%d", i)
|
||||
node, err := clientset.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
|
||||
// zone0: node0, node1
|
||||
// zone1: node2
|
||||
var zoneName string
|
||||
if i < 2 {
|
||||
zoneName = "zone0"
|
||||
} else {
|
||||
zoneName = "zone1"
|
||||
}
|
||||
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": zoneName}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
}
|
||||
|
||||
executor := &exectest.MockExecutor{}
|
||||
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
context := &clusterd.Context{
|
||||
Clientset: clientset,
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
|
||||
cluster := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(cluster, 2, cephv1.MonSpec{Count: 2, AllowMultiplePerNode: false}, "myversion")
|
||||
cluster.waitForStart = false
|
||||
defer os.RemoveAll(cluster.context.ConfigDir)
|
||||
|
||||
// add mon mappings
|
||||
cluster.mapping.Node["a"] = &NodeInfo{
|
||||
Name: "node0",
|
||||
Hostname: "mynode0",
|
||||
Address: "0.0.0.0",
|
||||
}
|
||||
cluster.mapping.Node["b"] = &NodeInfo{
|
||||
Name: "node1",
|
||||
Hostname: "mynode1",
|
||||
Address: "0.0.0.0",
|
||||
}
|
||||
cluster.maxMonID = 1
|
||||
cluster.saveMonConfig()
|
||||
|
||||
monNames := []string{"a", "b"}
|
||||
nodeNames := []string{"node0", "node1"}
|
||||
for i := 0; i < len(monNames); i++ {
|
||||
monConfig := testGenMonConfig(monNames[i])
|
||||
nodeName := nodeNames[i]
|
||||
|
||||
d := cluster.makeDeployment(monConfig, nodeName)
|
||||
_, err := clientset.AppsV1().Deployments(cluster.Namespace).Create(d)
|
||||
assert.Nil(t, err)
|
||||
|
||||
po := cluster.makeMonPod(monConfig, nodeName)
|
||||
_, err = clientset.CoreV1().Pods(cluster.Namespace).Create(po)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
assert.NotContains(t, cluster.mapping.Node, "c")
|
||||
assert.Equal(t, "node0", cluster.mapping.Node["a"].Name)
|
||||
assert.Equal(t, "node1", cluster.mapping.Node["b"].Name)
|
||||
assert.Equal(t, "mynode0", cluster.mapping.Node["a"].Hostname)
|
||||
assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname)
|
||||
|
||||
_, err := cluster.resolveInvalidMonitorPlacement(2)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// mon.a was chosen from the overloaded zone 0, and a new monitor to take
|
||||
// its place, mon.c was created on a node in zone 1 that had no monitors.
|
||||
assert.NotContains(t, cluster.mapping.Node, "a")
|
||||
assert.Equal(t, "node1", cluster.mapping.Node["b"].Name)
|
||||
assert.Equal(t, "node2", cluster.mapping.Node["c"].Name)
|
||||
assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname)
|
||||
}
|
||||
|
||||
func TestCheckHealthTwoMonsOneNode(t *testing.T) {
|
||||
executorNextMons := false
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
|
||||
if executorNextMons {
|
||||
resp := client.MonStatusResponse{Quorum: []int{0}}
|
||||
resp.MonMap.Mons = []client.MonMapEntry{
|
||||
{
|
||||
Name: "a",
|
||||
Rank: 0,
|
||||
Address: "1.2.3.4",
|
||||
},
|
||||
{
|
||||
Name: "c",
|
||||
Rank: 0,
|
||||
Address: "1.2.3.4",
|
||||
},
|
||||
}
|
||||
serialized, _ := json.Marshal(resp)
|
||||
return string(serialized), nil
|
||||
}
|
||||
return clienttest.MonInQuorumResponseMany(2), nil
|
||||
},
|
||||
}
|
||||
clientset := test.New(1)
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
context := &clusterd.Context{
|
||||
Clientset: clientset,
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
|
||||
c := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(c, 2, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
c.waitForStart = false
|
||||
defer os.RemoveAll(c.context.ConfigDir)
|
||||
|
||||
// add two mons to the mapping on node0
|
||||
c.mapping.Node["a"] = &NodeInfo{
|
||||
Name: "node0",
|
||||
Hostname: "mynode0",
|
||||
Address: "0.0.0.0",
|
||||
}
|
||||
c.mapping.Node["b"] = &NodeInfo{
|
||||
Name: "node0",
|
||||
Hostname: "mynode0",
|
||||
Address: "0.0.0.0",
|
||||
}
|
||||
c.maxMonID = 1
|
||||
c.saveMonConfig()
|
||||
|
||||
monNames := []string{"a", "b"}
|
||||
for i := 0; i < len(monNames); i++ {
|
||||
monConfig := testGenMonConfig(monNames[i])
|
||||
d := c.makeDeployment(monConfig, "node0")
|
||||
_, err := clientset.AppsV1().Deployments(c.Namespace).Create(d)
|
||||
assert.Nil(t, err)
|
||||
po := c.makeMonPod(monConfig, "node0")
|
||||
_, err = clientset.CoreV1().Pods(c.Namespace).Create(po)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
// initial health check should already see that there is more than one mon on one node (node0)
|
||||
_, err := c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "node0", c.mapping.Node["a"].Name)
|
||||
assert.Equal(t, "node0", c.mapping.Node["b"].Name)
|
||||
assert.Equal(t, "mynode0", c.mapping.Node["a"].Hostname)
|
||||
assert.Equal(t, "mynode0", c.mapping.Node["b"].Hostname)
|
||||
|
||||
// add new node and check if the second mon gets failovered to it
|
||||
n := &v1.Node{
|
||||
Status: v1.NodeStatus{
|
||||
Conditions: []v1.NodeCondition{
|
||||
{
|
||||
Type: v1.NodeReady, Status: v1.ConditionTrue,
|
||||
},
|
||||
},
|
||||
Addresses: []v1.NodeAddress{
|
||||
{
|
||||
Type: v1.NodeExternalIP,
|
||||
Address: "2.2.2.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
n.Name = "node2"
|
||||
clientset.CoreV1().Nodes().Create(n)
|
||||
|
||||
_, err = c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// check that mon c exists
|
||||
logger.Infof("mapping: %+v", c.mapping.Node)
|
||||
assert.NotNil(t, c.mapping.Node["c"])
|
||||
assert.Equal(t, "node2", c.mapping.Node["c"].Name)
|
||||
|
||||
// check if mon b has been deleted
|
||||
var dlist *apps.DeploymentList
|
||||
dlist, err = clientset.AppsV1().Deployments(c.Namespace).List(metav1.ListOptions{})
|
||||
assert.Nil(t, err)
|
||||
deleted := true
|
||||
for _, d := range dlist.Items {
|
||||
if d.Name == "b" {
|
||||
deleted = false
|
||||
}
|
||||
}
|
||||
assert.Equal(t, true, deleted, "mon b not failed over or deleted after health check")
|
||||
|
||||
// enable different ceph mon map output
|
||||
executorNextMons = true
|
||||
_, err = c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// check that nothing has changed
|
||||
dlist, err = clientset.AppsV1().Deployments(c.Namespace).List(metav1.ListOptions{})
|
||||
assert.Nil(t, err)
|
||||
|
||||
for _, d := range dlist.Items {
|
||||
// both mons should always be on the same node as in this test due to the order
|
||||
// the mons are processed in the loop
|
||||
if (d.Name == "rook-ceph-mon-a" && d.Spec.Template.Spec.NodeSelector[v1.LabelHostname] == "node1") || (d.Name != "rook-ceph-mon-c" && d.Spec.Template.Spec.NodeSelector[v1.LabelHostname] == "node2") {
|
||||
assert.Fail(t, fmt.Sprintf("mon %s shouldn't exist", d.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckMonsValid(t *testing.T) {
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
|
||||
return clienttest.MonInQuorumResponse(), nil
|
||||
},
|
||||
}
|
||||
clientset := test.New(1)
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
context := &clusterd.Context{
|
||||
Clientset: clientset,
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
c.waitForStart = false
|
||||
defer os.RemoveAll(c.context.ConfigDir)
|
||||
|
||||
// add two mons to the mapping on node0
|
||||
c.mapping.Node["a"] = &NodeInfo{
|
||||
Name: "node0",
|
||||
Address: "0.0.0.0",
|
||||
}
|
||||
c.mapping.Node["b"] = &NodeInfo{
|
||||
Name: "node1",
|
||||
Address: "0.0.0.0",
|
||||
}
|
||||
c.maxMonID = 1
|
||||
|
||||
// add three nodes
|
||||
for i := 0; i < 3; i++ {
|
||||
n := &v1.Node{
|
||||
Status: v1.NodeStatus{
|
||||
Conditions: []v1.NodeCondition{
|
||||
{
|
||||
Type: v1.NodeReady, Status: v1.ConditionTrue,
|
||||
},
|
||||
},
|
||||
Addresses: []v1.NodeAddress{
|
||||
{
|
||||
Type: v1.NodeExternalIP,
|
||||
Address: "0.0.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
n.Name = fmt.Sprintf("node%d", i)
|
||||
clientset.CoreV1().Nodes().Create(n)
|
||||
}
|
||||
|
||||
_, err := c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "node0", c.mapping.Node["a"].Name)
|
||||
assert.Equal(t, "node1", c.mapping.Node["b"].Name)
|
||||
|
||||
// set node0 to contain the not ready status condition, and check that mon.a
|
||||
// is failovered to be mon.c on node2. note that validity here doesn't include the
|
||||
// unscheduable flag (for mons already placed on a node).
|
||||
node0, err := c.context.Clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
node0.Status.Conditions[0].Status = v1.ConditionFalse
|
||||
_, err = c.context.Clientset.CoreV1().Nodes().Update(node0)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// add the pods so the getNodesInUse() works correctly
|
||||
monNames := []string{"a", "b", "c"}
|
||||
for i := 1; i <= 2; i++ { // 1=b, 2=c
|
||||
po := c.makeMonPod(testGenMonConfig(monNames[i]), fmt.Sprintf("node%d", i-1))
|
||||
_, err = clientset.CoreV1().Pods(c.Namespace).Create(po)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
_, err = c.resolveInvalidMonitorPlacement(3)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Len(t, c.mapping.Node, 2)
|
||||
logger.Infof("mapping: %+v", c.mapping)
|
||||
assert.Nil(t, c.mapping.Node["a"])
|
||||
// the new mon should always be on the empty node2
|
||||
// the failovered mon's name is "rook-ceph-mon-a"
|
||||
assert.Equal(t, "node2", c.mapping.Node["c"].Name)
|
||||
assert.Equal(t, "node1", c.mapping.Node["b"].Name)
|
||||
}
|
||||
|
||||
func TestAddRemoveMons(t *testing.T) {
|
||||
var deploymentsUpdated *[]*apps.Deployment
|
||||
updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
|
||||
|
||||
@@ -43,6 +43,7 @@ import (
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -83,10 +84,18 @@ const (
|
||||
// default storage request size for ceph monitor pvc
|
||||
// https://docs.ceph.com/docs/master/start/hardware-recommendations/#monitors-and-managers-ceph-mon-and-ceph-mgr
|
||||
cephMonDefaultStorageRequest = "10Gi"
|
||||
|
||||
// canary pod scheduling uses retry loops when cleaning up previous canary
|
||||
// pods and waiting for kubernetes scheduling to complete.
|
||||
canaryRetries = 180
|
||||
canaryRetryDelaySeconds = 5
|
||||
)
|
||||
|
||||
var (
|
||||
logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mon")
|
||||
|
||||
// hook for tests to override
|
||||
scheduleMonitor = realScheduleMonitor
|
||||
)
|
||||
|
||||
// Cluster represents the Rook and environment configuration settings needed to set up Ceph mons.
|
||||
@@ -129,7 +138,6 @@ type monConfig struct {
|
||||
// Mapping is mon node and port mapping
|
||||
type Mapping struct {
|
||||
Node map[string]*NodeInfo `json:"node"`
|
||||
Port map[string]int32 `json:"port"`
|
||||
}
|
||||
|
||||
// NodeInfo contains name and address of a node
|
||||
@@ -139,6 +147,12 @@ type NodeInfo struct {
|
||||
Address string
|
||||
}
|
||||
|
||||
type SchedulingResult struct {
|
||||
Node *v1.Node
|
||||
CanaryDeployment string
|
||||
CanaryPVC string
|
||||
}
|
||||
|
||||
// New creates an instance of a mon cluster
|
||||
func New(context *clusterd.Context, namespace, dataDirHostPath string, network cephv1.NetworkSpec, ownerRef metav1.OwnerReference, csiConfigMutex *sync.Mutex) *Cluster {
|
||||
return &Cluster{
|
||||
@@ -153,7 +167,6 @@ func New(context *clusterd.Context, namespace, dataDirHostPath string, network c
|
||||
Network: network,
|
||||
mapping: &Mapping{
|
||||
Node: map[string]*NodeInfo{},
|
||||
Port: map[string]int32{},
|
||||
},
|
||||
ownerRef: ownerRef,
|
||||
csiConfigMutex: csiConfigMutex,
|
||||
@@ -189,14 +202,10 @@ func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string,
|
||||
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
|
||||
}
|
||||
|
||||
targetCount, msg, err := c.getTargetMonCount()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get target mon count. %+v", err)
|
||||
}
|
||||
logger.Infof(msg)
|
||||
logger.Infof("targeting the mon count %d", c.spec.Mon.Count)
|
||||
|
||||
// create the mons for a new cluster or ensure mons are running in an existing cluster
|
||||
return c.ClusterInfo, c.startMons(targetCount)
|
||||
return c.ClusterInfo, c.startMons(c.spec.Mon.Count)
|
||||
}
|
||||
|
||||
func (c *Cluster) startMons(targetCount int) error {
|
||||
@@ -365,6 +374,136 @@ func resourceName(name string) string {
|
||||
return fmt.Sprintf("%s-%s", AppName, name)
|
||||
}
|
||||
|
||||
// scheduleMonitor selects a node for a monitor deployment.
|
||||
// see startMon() and design/ceph-mon-pv.md for additional details.
|
||||
func realScheduleMonitor(c *Cluster, mon *monConfig) (SchedulingResult, error) {
|
||||
// target node decision, and deployment/pvc to cleanup
|
||||
result := SchedulingResult{
|
||||
Node: nil,
|
||||
CanaryDeployment: "",
|
||||
CanaryPVC: "",
|
||||
}
|
||||
|
||||
// build the canary deployment.
|
||||
d := c.makeDeployment(mon)
|
||||
d.Name += "-canary"
|
||||
d.Spec.Template.ObjectMeta.Name += "-canary"
|
||||
|
||||
// the canary and real monitor deployments will mount the same storage. to
|
||||
// avoid issues with the real deployment, the canary should be careful not
|
||||
// to modify the storage by instead running an innocuous command.
|
||||
d.Spec.Template.Spec.InitContainers = []v1.Container{}
|
||||
d.Spec.Template.Spec.Containers[0].Image = c.rookVersion
|
||||
d.Spec.Template.Spec.Containers[0].Command = []string{"/tini"}
|
||||
d.Spec.Template.Spec.Containers[0].Args = []string{"--", "sleep", "3600"}
|
||||
|
||||
// setup affinity settings for pod scheduling
|
||||
p := cephv1.GetMonPlacement(c.spec.Placement)
|
||||
c.setPodPlacement(&d.Spec.Template.Spec, p, nil)
|
||||
|
||||
// setup storage on the canary since scheduling will be affected when
|
||||
// monitors are configured to use persistent volumes. the pvcName is set to
|
||||
// the non-empty name of the PVC only when the PVC is created as a result of
|
||||
// this call to the scheduler.
|
||||
var pvcName string
|
||||
if c.spec.Mon.VolumeClaimTemplate == nil {
|
||||
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes,
|
||||
opspec.DaemonVolumesDataHostPath(mon.DataPathMap)...)
|
||||
} else {
|
||||
// the pvc that is created here won't be deleted: it will be reattached
|
||||
// to the real monitor deployment.
|
||||
pvc, err := c.makeDeploymentPVC(mon)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("sched-mon: failed to make monitor %s pvc. %+v", d.Name, err)
|
||||
}
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(pvc)
|
||||
if err == nil {
|
||||
pvcName = pvc.Name
|
||||
logger.Infof("sched-mon: created canary monitor %s pvc %s", d.Name, pvc.Name)
|
||||
} else {
|
||||
if errors.IsAlreadyExists(err) {
|
||||
logger.Debugf("sched-mon: creating mon %s pvc %s: already exists.", d.Name, pvc.Name)
|
||||
} else {
|
||||
return result, fmt.Errorf("sched-mon: error creating mon %s pvc %s. %+v", d.Name, pvc.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes,
|
||||
opspec.DaemonVolumesDataPVC(mon.ResourceName))
|
||||
opspec.AddVolumeMountSubPath(&d.Spec.Template.Spec, "ceph-daemon-data")
|
||||
}
|
||||
|
||||
// caller should arrange for clean-up of the PVC only if it was created
|
||||
// during this scheduling instance and we encounter an irrecoverable error.
|
||||
result.CanaryPVC = pvcName
|
||||
|
||||
// spin up the canary deployment. if it exists, delete it first, since if it
|
||||
// already exists it may have been scheduled with a different crd config.
|
||||
createdDeployment := false
|
||||
for i := 0; i < canaryRetries; i++ {
|
||||
_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d)
|
||||
if err == nil {
|
||||
createdDeployment = true
|
||||
logger.Infof("sched-mon: created canary deployment %s", d.Name)
|
||||
break
|
||||
} else if errors.IsAlreadyExists(err) {
|
||||
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, d.Name); err != nil {
|
||||
return result, fmt.Errorf("sched-mon: error deleting canary deployment %s. %+v", d.Name, err)
|
||||
}
|
||||
logger.Infof("sched-mon: deleted existing canary deployment %s", d.Name)
|
||||
time.Sleep(time.Second * canaryRetryDelaySeconds)
|
||||
} else {
|
||||
return result, fmt.Errorf("sched-mon: error creating canary monitor deployment %s. %+v", d.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// failed after retrying
|
||||
if !createdDeployment {
|
||||
return result, fmt.Errorf("sched-mon: failed to create canary deployment %s", d.Name)
|
||||
}
|
||||
|
||||
// caller should arrange for the deployment to be removed
|
||||
result.CanaryDeployment = d.Name
|
||||
|
||||
// wait for the scheduler to make a placement decision
|
||||
for i := 0; i < canaryRetries; i++ {
|
||||
listOptions := metav1.ListOptions{
|
||||
LabelSelector: labels.Set(d.Spec.Selector.MatchLabels).String(),
|
||||
}
|
||||
|
||||
pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(listOptions)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("sched-mon: error listing canary pods %s. %+v", d.Name, err)
|
||||
}
|
||||
|
||||
if len(pods.Items) == 0 {
|
||||
logger.Infof("sched-mon: waiting for canary pod creation %s", d.Name)
|
||||
time.Sleep(time.Second * canaryRetryDelaySeconds)
|
||||
continue
|
||||
}
|
||||
|
||||
pod := pods.Items[0]
|
||||
if pod.Spec.NodeName == "" {
|
||||
logger.Debugf("sched-mon: monitor %s canary pod %s not yet scheduled", d.Name, pod.Name)
|
||||
time.Sleep(time.Second * canaryRetryDelaySeconds)
|
||||
continue
|
||||
}
|
||||
|
||||
node, err := c.context.Clientset.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("sched-mon: error getting node %s. %+v", pod.Spec.NodeName, err)
|
||||
}
|
||||
|
||||
result.Node = node
|
||||
result.CanaryPVC = ""
|
||||
logger.Infof("sched-mon: canary monitor deployment %s scheduled to %s", d.Name, node.Name)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
return result, fmt.Errorf("sched-mon: canary pod scheduling failed retries")
|
||||
}
|
||||
|
||||
func (c *Cluster) initMonIPs(mons []*monConfig) error {
|
||||
for _, m := range mons {
|
||||
if c.Network.IsHost() {
|
||||
@@ -387,129 +526,82 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// find a suitable node for this monitor. first look for a fault zone that has
|
||||
// no monitors. only after the nodes with zone labels are considered are the
|
||||
// unlabeled nodes considered. reasonsing: without more information, _at worst_
|
||||
// the unlabled nodes are actually in the set of labeled zones, and at best,
|
||||
// they are in distinct zones.
|
||||
func scheduleMonitor(mon *monConfig, nodeZones [][]NodeUsage) *NodeUsage {
|
||||
// the node choice for this monitor
|
||||
var nodeChoice *NodeUsage
|
||||
|
||||
// for each zone, in order of preference. unlabeled nodes are last, by
|
||||
// construction; see Cluster.getNodeMonusage().
|
||||
for zi := range nodeZones {
|
||||
// number of monitors in the zone
|
||||
zoneMonCount := 0
|
||||
|
||||
// preferential node choice from this zone for a new mon
|
||||
var zoneNodeChoice *NodeUsage
|
||||
|
||||
// for each node in this zone
|
||||
for ni := range nodeZones[zi] {
|
||||
nodeUsage := &nodeZones[zi][ni]
|
||||
|
||||
// skip invalid nodes. but update the mon count first since
|
||||
// invalid nodes could still have an assigned mon pod.
|
||||
zoneMonCount += nodeUsage.MonCount
|
||||
if !nodeUsage.MonValid {
|
||||
logger.Infof("schedmon: skip invalid node %s for mon scheduling",
|
||||
nodeUsage.Node.Name)
|
||||
continue
|
||||
func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
// when monitors are scheduling below by invoking scheduleMonitor() a canary
|
||||
// deployment and optional canary PVC are created. in order for the
|
||||
// anti-affinity rules to be effective, we leave the canary pods in place
|
||||
// until all of the canaries have been scheduled. only after the
|
||||
// monitor/node assignment process is complete are the canary deployments
|
||||
// and pvcs removed here.
|
||||
canaryCleanup := []SchedulingResult{}
|
||||
defer func() {
|
||||
for _, result := range canaryCleanup {
|
||||
logger.Infof("assignmon: cleaning up canary deployment %s and canary pvc %s", result.CanaryDeployment, result.CanaryPVC)
|
||||
if result.CanaryDeployment != "" {
|
||||
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, result.CanaryDeployment); err != nil {
|
||||
logger.Infof("assignmon: error deleting canary deployment %s. %+v", result.CanaryDeployment, err)
|
||||
}
|
||||
}
|
||||
|
||||
// make a "best" choice from this zone. in this case that is the
|
||||
// node with the least amount of monitors.
|
||||
if zoneNodeChoice == nil || nodeUsage.MonCount < zoneNodeChoice.MonCount {
|
||||
logger.Infof("schedmon: considering node %s with mon count %d",
|
||||
nodeUsage.Node.Name, nodeUsage.MonCount)
|
||||
zoneNodeChoice = nodeUsage
|
||||
}
|
||||
}
|
||||
|
||||
// We identified _a_ valid node in the zone. should we choose it?
|
||||
if zoneNodeChoice != nil {
|
||||
if zoneMonCount == 0 {
|
||||
// this zone has no monitors, which implies that
|
||||
// zoneNodeChoice will be a node without any monitors
|
||||
// currently assigned. choose this node.
|
||||
logger.Infof("schedmon: considering node %s from empty zone",
|
||||
zoneNodeChoice.Node.Name)
|
||||
nodeChoice = zoneNodeChoice
|
||||
break
|
||||
} else {
|
||||
// this zone already has a monitor in it. but keep the
|
||||
// choice as a backup; we may find that all zones already
|
||||
// have a monitor and still need to make an assignment.
|
||||
if nodeChoice == nil || zoneNodeChoice.MonCount < nodeChoice.MonCount {
|
||||
logger.Infof("schedmon: considering node %s with mon count %d",
|
||||
zoneNodeChoice.Node.Name, zoneNodeChoice.MonCount)
|
||||
nodeChoice = zoneNodeChoice
|
||||
if result.CanaryPVC != "" {
|
||||
var gracePeriod int64 // delete immediately
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(result.CanaryPVC, options)
|
||||
if err != nil {
|
||||
logger.Infof("assignmon: error removing canary monitor %s pvc %s. %+v", result.CanaryDeployment, result.CanaryPVC, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if nodeChoice != nil {
|
||||
logger.Infof("schedmon: scheduling mon %s on node %s",
|
||||
mon.DaemonName, nodeChoice.Node.Name)
|
||||
} else {
|
||||
logger.Infof("schedmon: no suitable node found for mon %s",
|
||||
mon.DaemonName)
|
||||
}
|
||||
|
||||
return nodeChoice
|
||||
}
|
||||
|
||||
func (c *Cluster) assignMons(mons []*monConfig) error {
|
||||
|
||||
// retrieve the set of cluster nodes and their monitor usage info
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get node monitor usage. %+v", err)
|
||||
}
|
||||
|
||||
// ensure all monitors have a node assignment. note that this isn't
|
||||
// necessarily optimal: it does not try to move existing monitors which is
|
||||
// handled by the periodic monitor health checks.
|
||||
// ensure that all monitors have either (1) a node assignment that will be
|
||||
// enforced using a node selector, or (2) configuration permits k8s to handle
|
||||
// scheduling for the monitor.
|
||||
for _, mon := range mons {
|
||||
|
||||
// monitor is already assigned to a node. nothing to do
|
||||
// scheduling for this monitor has already been completed
|
||||
if _, ok := c.mapping.Node[mon.DaemonName]; ok {
|
||||
logger.Infof("assignmon: mon %s already assigned to a node", mon.DaemonName)
|
||||
logger.Debugf("assignmon: mon %s already scheduled", mon.DaemonName)
|
||||
continue
|
||||
}
|
||||
|
||||
nodeChoice := scheduleMonitor(mon, nodeZones)
|
||||
// determine a placement for the monitor. note that this scheduling is
|
||||
// performed even when a node selector is not required. this may be
|
||||
// non-optimal, but it is convenient to catch some failures early,
|
||||
// before a decision is stored in the node mapping.
|
||||
result, err := scheduleMonitor(c, mon)
|
||||
|
||||
if nodeChoice == nil {
|
||||
return fmt.Errorf("assignmon: no valid nodes available for mon placement")
|
||||
}
|
||||
// even if an error occurs cleanup may be necessary
|
||||
canaryCleanup = append(canaryCleanup, result)
|
||||
|
||||
// note that we do not need to worry about a false negative here (i.e. a
|
||||
// node exists that _would_ pass this check) due to the search landing in a
|
||||
// local minima because if a node had existed with a mon count of zero,
|
||||
// scheduleMonitor would have chosen it over any node with a positive
|
||||
// mon count.
|
||||
if nodeChoice.MonCount > 0 && !c.spec.Mon.AllowMultiplePerNode {
|
||||
return fmt.Errorf("assignmon: no empty nodes available for mon placement")
|
||||
}
|
||||
|
||||
// make this decision visible when scheduling the next monitor
|
||||
nodeChoice.MonCount++
|
||||
|
||||
logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Node.Name)
|
||||
|
||||
nodeInfo, err := getNodeInfoFromNode(*nodeChoice.Node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't get node info from node %s. %+v",
|
||||
nodeChoice.Node.Name, err)
|
||||
return fmt.Errorf("assignmon: error scheduling monitor: %+v", err)
|
||||
}
|
||||
|
||||
nodeChoice := result.Node
|
||||
if nodeChoice == nil {
|
||||
return fmt.Errorf("assignmon: could not schedule monitor %s", mon.DaemonName)
|
||||
}
|
||||
|
||||
// store nil in the node mapping to indicate that an explicit node
|
||||
// placement is not being made. otherwise, the node choice will map
|
||||
// directly to a node selector on the monitor pod.
|
||||
var nodeInfo *NodeInfo = nil
|
||||
if c.Network.IsHost() || c.spec.Mon.VolumeClaimTemplate == nil {
|
||||
logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name)
|
||||
nodeInfo, err = getNodeInfoFromNode(*nodeChoice)
|
||||
if err != nil {
|
||||
return fmt.Errorf("assignmon: couldn't get node info for node %s. %+v",
|
||||
nodeChoice.Name, err)
|
||||
}
|
||||
} else {
|
||||
logger.Infof("assignmon: mon %s placement using native scheduler", mon.DaemonName)
|
||||
}
|
||||
|
||||
c.mapping.Node[mon.DaemonName] = nodeInfo
|
||||
}
|
||||
|
||||
logger.Debug("assignmons: mons have been assigned to nodes")
|
||||
logger.Debug("assignmons: mons have been scheduled")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -521,7 +613,7 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
|
||||
// Ensure each of the mons have been created. If already created, it will be a no-op.
|
||||
for i := 0; i < len(mons); i++ {
|
||||
node, _ := c.mapping.Node[mons[i].DaemonName]
|
||||
err := c.startMon(mons[i], node.Hostname)
|
||||
err := c.startMon(mons[i], node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create mon %s. %+v", mons[i].DaemonName, err)
|
||||
}
|
||||
@@ -652,13 +744,42 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) startMon(m *monConfig, hostname string) error {
|
||||
|
||||
// startMon creates or updates a monitor deployment.
|
||||
//
|
||||
// The node parameter specifies the node to be used as a node selector on the
|
||||
// monitor pod. It is the result of scheduling a canary pod: see
|
||||
// scheduleMonitor() for more details on scheduling.
|
||||
//
|
||||
// The node parameter is optional. When the parameter is nil it indicates that
|
||||
// the pod should not use a node selector, and should instead rely on k8s to
|
||||
// perform scheduling.
|
||||
//
|
||||
// The following outlines the different scenarios that exist and how deployments
|
||||
// should be configured w.r.t. scheduling and the use of a node selector.
|
||||
//
|
||||
// 1) if HostNetworking -> always use node selector. we do not want to change
|
||||
// the IP address of a monitor as it is wrapped up in the monitor's identity.
|
||||
// with host networking we use node selector to ensure a stable IP for each
|
||||
// monitor. see scheduleMonitor() comment for more details.
|
||||
//
|
||||
// Note: an important assumption is that HostNetworking setting does not
|
||||
// change once a cluster is created.
|
||||
//
|
||||
// 2) if *not* HostNetworking -> stable IP from service; may avoid node selector
|
||||
// a) when creating a new deployment
|
||||
// - if HostPath -> use node selector for storage/node affinity
|
||||
// - if PVC -> node selector is not required
|
||||
//
|
||||
// b) when updating a deployment
|
||||
// - if HostPath -> leave node selector as is
|
||||
// - if PVC -> remove node selector, if present
|
||||
//
|
||||
func (c *Cluster) startMon(m *monConfig, node *NodeInfo) error {
|
||||
// check if the monitor deployment already exists. if the deployment does
|
||||
// exist, also determine if it using pvc storage.
|
||||
pvcExists := false
|
||||
deploymentExists := false
|
||||
d := c.makeDeployment(m, hostname)
|
||||
d := c.makeDeployment(m)
|
||||
existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.Name, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
deploymentExists = true
|
||||
@@ -683,7 +804,22 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error {
|
||||
logger.Debugf("adding host path volume source to mon deployment %s", d.Name)
|
||||
}
|
||||
|
||||
// placement settings from the CRD
|
||||
p := cephv1.GetMonPlacement(c.spec.Placement)
|
||||
|
||||
if deploymentExists {
|
||||
// the existing deployment may have a node selector. if the cluster
|
||||
// isn't using host networking and the deployment is using pvc storage,
|
||||
// then the node selector can be removed. this may happen after
|
||||
// upgrading the cluster with the k8s scheduling support for monitors.
|
||||
if c.Network.IsHost() || !pvcExists {
|
||||
p.PodAffinity = nil
|
||||
p.PodAntiAffinity = nil
|
||||
c.setPodPlacement(&d.Spec.Template.Spec, p,
|
||||
existingDeployment.Spec.Template.Spec.NodeSelector)
|
||||
} else {
|
||||
c.setPodPlacement(&d.Spec.Template.Spec, p, nil)
|
||||
}
|
||||
return c.updateMon(m, d)
|
||||
}
|
||||
|
||||
@@ -702,6 +838,15 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error {
|
||||
}
|
||||
}
|
||||
|
||||
if node == nil {
|
||||
c.setPodPlacement(&d.Spec.Template.Spec, p, nil)
|
||||
} else {
|
||||
p.PodAffinity = nil
|
||||
p.PodAntiAffinity = nil
|
||||
c.setPodPlacement(&d.Spec.Template.Spec, p,
|
||||
map[string]string{v1.LabelHostname: node.Hostname})
|
||||
}
|
||||
|
||||
logger.Debugf("Starting mon: %+v", d.Name)
|
||||
_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d)
|
||||
if err != nil {
|
||||
|
||||
@@ -122,7 +122,6 @@ func newCluster(context *clusterd.Context, namespace string, network cephv1.Netw
|
||||
monTimeoutList: map[string]time.Time{},
|
||||
mapping: &Mapping{
|
||||
Node: map[string]*NodeInfo{},
|
||||
Port: map[string]int32{},
|
||||
},
|
||||
ownerRef: metav1.OwnerReference{},
|
||||
}
|
||||
@@ -241,7 +240,7 @@ func TestSaveMonEndpoints(t *testing.T) {
|
||||
cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "a=1.2.3.1:6789", cm.Data[EndpointDataKey])
|
||||
assert.Equal(t, `{"node":{},"port":{}}`, cm.Data[MappingKey])
|
||||
assert.Equal(t, `{"node":{}}`, cm.Data[MappingKey])
|
||||
assert.Equal(t, "-1", cm.Data[MaxMonIDKey])
|
||||
|
||||
// update the config map
|
||||
@@ -252,14 +251,13 @@ func TestSaveMonEndpoints(t *testing.T) {
|
||||
Address: "1.1.1.1",
|
||||
Hostname: "myhost",
|
||||
}
|
||||
c.mapping.Port["node0"] = int32(12345)
|
||||
err = c.saveMonConfig()
|
||||
assert.Nil(t, err)
|
||||
|
||||
cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "a=2.3.4.5:6789", cm.Data[EndpointDataKey])
|
||||
assert.Equal(t, `{"node":{"a":{"Name":"node0","Hostname":"myhost","Address":"1.1.1.1"}},"port":{"node0":12345}}`, cm.Data[MappingKey])
|
||||
assert.Equal(t, `{"node":{"a":{"Name":"node0","Hostname":"myhost","Address":"1.1.1.1"}}}`, cm.Data[MappingKey])
|
||||
assert.Equal(t, "2", cm.Data[MaxMonIDKey])
|
||||
}
|
||||
|
||||
@@ -344,168 +342,3 @@ func TestMonFoundInQuorum(t *testing.T) {
|
||||
assert.True(t, monFoundInQuorum("c", response))
|
||||
assert.False(t, monFoundInQuorum("d", response))
|
||||
}
|
||||
|
||||
// no node choice can be made when there are no nodes
|
||||
func TestScheduleMonitorEmpty(t *testing.T) {
|
||||
nodeZones := [][]NodeUsage{}
|
||||
mon := &monConfig{DaemonName: "a"}
|
||||
// no zones
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
// 1 zone no mons
|
||||
nodeZones = append(nodeZones, []NodeUsage{})
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
// 2 zones no mons
|
||||
nodeZones = append(nodeZones, []NodeUsage{})
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
}
|
||||
|
||||
// only valid nodes should be chosen
|
||||
func TestScheduleMonitorInvalidNodes(t *testing.T) {
|
||||
// 1 zone with 1 invalid empty node
|
||||
nodeZones := [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
},
|
||||
}
|
||||
mon := &monConfig{DaemonName: "a"}
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// 1 zone with 2 invalid empty node
|
||||
nodeZones = [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
},
|
||||
}
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// 2 zone with 2 invalid empty node each
|
||||
nodeZones = [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
},
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false},
|
||||
},
|
||||
}
|
||||
assert.Nil(t, scheduleMonitor(mon, nodeZones))
|
||||
}
|
||||
|
||||
func TestScheduleMonitor(t *testing.T) {
|
||||
// 1 zone, 1 valid, empty node -> only one choice
|
||||
nodeZones := [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
}
|
||||
mon := &monConfig{DaemonName: "a"}
|
||||
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// still the only choice even if not empty
|
||||
nodeZones = [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
|
||||
},
|
||||
}
|
||||
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// scheduler prefers the node with the least number of mons
|
||||
nodeZones = [][]NodeUsage{
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
|
||||
},
|
||||
}
|
||||
assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// the scheduler prefers nodes with the least number of mons, not zones with
|
||||
// the leads number of mons (zero mons is a special case: tested below...)
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 24 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 4, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true},
|
||||
},
|
||||
// 10 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the node with 4 mons
|
||||
assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// same as before, the target mon is in the second zone
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 6 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true},
|
||||
},
|
||||
// 10 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 9, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the node with 1 mon
|
||||
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// prefers a zone with zero mons to spread across failure domains
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 0 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
// 1 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the zone with zero mons
|
||||
assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// same as before, different zone
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 0 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
// 1 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the zone with zero mons
|
||||
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
|
||||
|
||||
// invalid nodes aren't schedulable, but if they have mons, that factors
|
||||
// into the decision. in this case it means the zone isn't really empty
|
||||
nodeZones = [][]NodeUsage{
|
||||
// 0 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
// invalid node, but has a mon -> zone is not empty
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: false},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
// 1 mons
|
||||
{
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true},
|
||||
},
|
||||
}
|
||||
// choose the zone with zero mons
|
||||
assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones))
|
||||
}
|
||||
@@ -19,11 +19,7 @@ package mon
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
// NodeUsage is a mapping between a Node and computed metadata about the node
|
||||
@@ -37,139 +33,6 @@ type NodeUsage struct {
|
||||
MonValid bool
|
||||
}
|
||||
|
||||
// getMonNodes detects the nodes that are available for new mons to start.
|
||||
func (c *Cluster) getMonNodes() ([]v1.Node, error) {
|
||||
availableNodes, nodes, err := c.getAvailableMonNodes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger.Infof("Found %d running nodes without mons", len(availableNodes))
|
||||
|
||||
// if all nodes already have mons and the user has given the mon.count, add all nodes to be available
|
||||
if c.spec.Mon.AllowMultiplePerNode && len(availableNodes) == 0 {
|
||||
logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items))
|
||||
for _, node := range nodes.Items {
|
||||
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
availableNodes = append(availableNodes, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return availableNodes, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) {
|
||||
nodeOptions := metav1.ListOptions{}
|
||||
nodeOptions.TypeMeta.Kind = "Node"
|
||||
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
logger.Debugf("there are %d nodes available for %d mons", len(nodes.Items), len(c.ClusterInfo.Monitors))
|
||||
|
||||
// get the nodes that have mons assigned
|
||||
nodesInUse, err := c.getNodesWithMons(nodes)
|
||||
if err != nil {
|
||||
logger.Warningf("could not get nodes with mons. %+v", err)
|
||||
nodesInUse = util.NewSet()
|
||||
}
|
||||
|
||||
// choose nodes for the new mons that don't have mons currently
|
||||
availableNodes := []v1.Node{}
|
||||
for _, node := range nodes.Items {
|
||||
if !nodesInUse.Contains(node.Name) {
|
||||
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
availableNodes = append(availableNodes, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return availableNodes, nodes, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) getNodesWithMons(nodes *v1.NodeList) (*util.Set, error) {
|
||||
// get the mon pods and their node affinity
|
||||
options := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}
|
||||
pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodesInUse := util.NewSet()
|
||||
for _, pod := range pods.Items {
|
||||
hostname := pod.Spec.NodeSelector[v1.LabelHostname]
|
||||
logger.Debugf("mon pod on node %s", hostname)
|
||||
name, ok := getNodeNameFromHostname(nodes, hostname)
|
||||
if !ok {
|
||||
logger.Errorf("mon %s on hostname %s not found in node list", pod.Name, hostname)
|
||||
}
|
||||
nodesInUse.Add(name)
|
||||
}
|
||||
return nodesInUse, nil
|
||||
}
|
||||
|
||||
// Get the number of mons that the operator should be starting
|
||||
func (c *Cluster) getTargetMonCount() (int, string, error) {
|
||||
|
||||
// Get the full list of k8s nodes
|
||||
nodeOptions := metav1.ListOptions{}
|
||||
nodeOptions.TypeMeta.Kind = "Node"
|
||||
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("failed to get nodes. %+v", err)
|
||||
}
|
||||
|
||||
// Get the nodes where it is possible to place mons
|
||||
availableNodes := []v1.Node{}
|
||||
for _, node := range nodes.Items {
|
||||
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
availableNodes = append(availableNodes, node)
|
||||
}
|
||||
}
|
||||
|
||||
target, msg := calcTargetMonCount(len(availableNodes), c.spec.Mon)
|
||||
return target, msg, nil
|
||||
}
|
||||
|
||||
func calcTargetMonCount(nodes int, spec cephv1.MonSpec) (int, string) {
|
||||
minTarget := spec.Count
|
||||
preferredTarget := spec.PreferredCount
|
||||
|
||||
if preferredTarget <= minTarget {
|
||||
msg := fmt.Sprintf("targeting the mon count %d", minTarget)
|
||||
return minTarget, msg
|
||||
}
|
||||
if nodes >= preferredTarget {
|
||||
msg := fmt.Sprintf("targeting the preferred mon count %d since there are %d available nodes", preferredTarget, nodes)
|
||||
return preferredTarget, msg
|
||||
}
|
||||
if spec.AllowMultiplePerNode {
|
||||
msg := fmt.Sprintf("targeting the preferred mon count %d even if not that many nodes since multiple mons are allowed per node", preferredTarget)
|
||||
return preferredTarget, msg
|
||||
}
|
||||
if nodes <= minTarget {
|
||||
msg := fmt.Sprintf("targeting the min mon count %d since there are only %d available nodes", minTarget, nodes)
|
||||
return minTarget, msg
|
||||
}
|
||||
|
||||
// There are between minTarget and preferredTarget nodes. Find an odd number for mons closest to the number of nodes.
|
||||
intermediate := nodes
|
||||
if intermediate%2 == 0 {
|
||||
// Decrease to an odd number if there are an even number of nodes
|
||||
intermediate--
|
||||
}
|
||||
msg := fmt.Sprintf("targeting the calculated mon count %d since there are %d available nodes", intermediate, nodes)
|
||||
return intermediate, msg
|
||||
}
|
||||
|
||||
// Look up the immutable node name from the hostname label
|
||||
func getNodeNameFromHostname(nodes *v1.NodeList, hostname string) (string, bool) {
|
||||
for _, node := range nodes.Items {
|
||||
@@ -201,76 +64,3 @@ func getNodeInfoFromNode(n v1.Node) (*NodeInfo, error) {
|
||||
}
|
||||
return nr, nil
|
||||
}
|
||||
|
||||
// Returns a hierarchical representation of the nodes in the cluster organized
|
||||
// by zone failure domain and annotated with the number of monitor pods assigned
|
||||
// to each node and if the node is valid for scheduling monitor pods. The
|
||||
// purpose of this structure is to provide a unified view of information
|
||||
// required for making monitor scheduling decisions.
|
||||
func (c *Cluster) getNodeMonUsage() ([][]NodeUsage, error) {
|
||||
// get all k8s node objects
|
||||
nodeOptions := metav1.ListOptions{}
|
||||
nodeOptions.TypeMeta.Kind = "Node"
|
||||
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// get all pod objects labeled as a monitor
|
||||
podOptions := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}
|
||||
pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(podOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// generate a list of nodes that includes the number of monitor pods
|
||||
// assigned to each node. this is equivalent to:
|
||||
//
|
||||
// SELECT Node, Count(*) FROM Nodes
|
||||
// JOIN Pods ON Nodes.Hostname == Pods.Hostname
|
||||
// GROUP BY Node
|
||||
//
|
||||
nodeUsages := []NodeUsage{}
|
||||
for i, node := range nodes.Items {
|
||||
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
continue
|
||||
}
|
||||
nodeUsage := NodeUsage{Node: &nodes.Items[i], MonCount: 0, MonValid: valid}
|
||||
for _, pod := range pods.Items {
|
||||
hostname := pod.Spec.NodeSelector[v1.LabelHostname]
|
||||
if node.Name == hostname || node.Labels[v1.LabelHostname] == hostname {
|
||||
nodeUsage.MonCount++
|
||||
}
|
||||
}
|
||||
nodeUsages = append(nodeUsages, nodeUsage)
|
||||
}
|
||||
|
||||
// generate a hierarchical representation of the nodes grouped by the name
|
||||
// of the node's zone failure domain label. the zone label named "" is used
|
||||
// for nodes without a zone annotation.
|
||||
nodesByZone := make(map[string][]NodeUsage)
|
||||
for _, nodeUsage := range nodeUsages {
|
||||
zone, ok := nodeUsage.Node.Labels["failure-domain.beta.kubernetes.io/zone"]
|
||||
if !ok {
|
||||
zone = ""
|
||||
}
|
||||
nodesByZone[zone] = append(nodesByZone[zone], nodeUsage)
|
||||
}
|
||||
|
||||
// compute the final form used for scheduling. this form is an array of
|
||||
// nodes by zone, with non-labeled nodes appearing in the final zone. this
|
||||
// organization reflects the preference made by the scheduler of using nodes
|
||||
// with zone annotations before non-labeled nodes.
|
||||
res := [][]NodeUsage{}
|
||||
for zone, nodeUsage := range nodesByZone {
|
||||
if zone == "" {
|
||||
res = append(res, nodeUsage)
|
||||
} else {
|
||||
res = append([][]NodeUsage{nodeUsage}, res...)
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
@@ -32,110 +32,6 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestGetNodeMonUsageValidNode(t *testing.T) {
|
||||
clientset := test.New(2)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
conditions := v1.NodeCondition{Type: v1.NodeOutOfDisk}
|
||||
node.Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// nodes are reported with correct valid flag
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, nodeZones[0][0].MonValid)
|
||||
assert.True(t, nodeZones[0][1].MonValid)
|
||||
}
|
||||
|
||||
func TestGetNodeMonUsageMonCount(t *testing.T) {
|
||||
clientset := test.New(2)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
// 3 mons on node0
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
|
||||
monIDs := []string{"a", "b", "c"}
|
||||
for i := 0; i < 3; i++ {
|
||||
pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name)
|
||||
_, err = clientset.CoreV1().Pods(c.Namespace).Create(pod)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
// 2 mons on node1
|
||||
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
|
||||
monIDs = []string{"d", "e"}
|
||||
for i := 0; i < 2; i++ {
|
||||
pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name)
|
||||
_, err = clientset.CoreV1().Pods(c.Namespace).Create(pod)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(nodeZones))
|
||||
if nodeZones[0][0].Node.Name == "node0" {
|
||||
assert.Equal(t, 3, nodeZones[0][0].MonCount)
|
||||
assert.Equal(t, 2, nodeZones[0][1].MonCount)
|
||||
} else {
|
||||
assert.Equal(t, 2, nodeZones[0][1].MonCount)
|
||||
assert.Equal(t, 3, nodeZones[0][0].MonCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaintedNodes(t *testing.T) {
|
||||
clientset := test.New(4)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
// mark a node as unschedulable
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Spec.Unschedulable = true
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// taint
|
||||
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Spec.Taints = []v1.Taint{
|
||||
{Effect: v1.TaintEffectNoSchedule},
|
||||
}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// taint
|
||||
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Spec.Taints = []v1.Taint{
|
||||
{Effect: v1.TaintEffectPreferNoSchedule},
|
||||
}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(nodeZones))
|
||||
validCount := 0
|
||||
for zi := range nodeZones {
|
||||
for ni := range nodeZones[zi] {
|
||||
if nodeZones[zi][ni].MonValid {
|
||||
validCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 1, validCount)
|
||||
}
|
||||
|
||||
func TestNodeAffinity(t *testing.T) {
|
||||
clientset := test.New(4)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
@@ -160,30 +56,17 @@ func TestNodeAffinity(t *testing.T) {
|
||||
}
|
||||
|
||||
// label nodes so they appear as not scheduable / invalid
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
node, _ := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
node.Labels = map[string]string{"label": "foo"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
node, _ = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
node.Labels = map[string]string{"label": "bar"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
|
||||
node, _ = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
|
||||
node.Labels = map[string]string{"label": "baz"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(nodeZones))
|
||||
validCount := 0
|
||||
for zi := range nodeZones {
|
||||
for ni := range nodeZones[zi] {
|
||||
if nodeZones[zi][ni].MonValid {
|
||||
validCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 2, validCount)
|
||||
}
|
||||
|
||||
// this tests can 3 mons with hostnetworking on the same host is rejected
|
||||
@@ -282,22 +165,8 @@ func TestHostNetwork(t *testing.T) {
|
||||
|
||||
c.Network.HostNetwork = true
|
||||
|
||||
nodes := []v1.Node{}
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.Nil(t, err)
|
||||
|
||||
for zi := range nodeZones {
|
||||
for _, nodeUsage := range nodeZones[zi] {
|
||||
if nodeUsage.MonCount == 0 || c.spec.Mon.AllowMultiplePerNode {
|
||||
nodes = append(nodes, *nodeUsage.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, 3, len(nodes))
|
||||
|
||||
monConfig := testGenMonConfig("c")
|
||||
pod := c.makeMonPod(monConfig, nodes[2].Name)
|
||||
pod := c.makeMonPod(monConfig)
|
||||
assert.NotNil(t, pod)
|
||||
assert.Equal(t, true, pod.Spec.HostNetwork)
|
||||
assert.Equal(t, v1.DNSClusterFirstWithHostNet, pod.Spec.DNSPolicy)
|
||||
@@ -308,7 +177,7 @@ func TestHostNetwork(t *testing.T) {
|
||||
assert.Equal(t, "arg not found: --public-bind-addr", message)
|
||||
|
||||
monConfig.Port = 6790
|
||||
pod = c.makeMonPod(monConfig, nodes[2].Name)
|
||||
pod = c.makeMonPod(monConfig)
|
||||
val, message = extractArgValue(pod.Spec.Containers[0].Args, "--public-addr")
|
||||
assert.Equal(t, "2.4.6.3:6790", val, message)
|
||||
assert.NotNil(t, pod)
|
||||
@@ -347,186 +216,3 @@ func TestGetNodeInfoFromNode(t *testing.T) {
|
||||
|
||||
assert.Equal(t, "1.1.1.1", info.Address)
|
||||
}
|
||||
|
||||
func TestTargetMonCount(t *testing.T) {
|
||||
// only a single node and min 1
|
||||
spec := cephv1.MonSpec{Count: 1, PreferredCount: 0, AllowMultiplePerNode: false}
|
||||
nodes := 1
|
||||
target, msg := calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 1, target)
|
||||
logger.Infof(msg)
|
||||
|
||||
// preferred 3 mons, but only one node available
|
||||
spec.PreferredCount = 3
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 1, target)
|
||||
logger.Infof(msg)
|
||||
|
||||
// preferred 3 mons, and 3 nodes available
|
||||
nodes = 3
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 3, target)
|
||||
logger.Infof(msg)
|
||||
|
||||
// get an intermediate odd number since we have several options between the min and the node count
|
||||
nodes = 4
|
||||
spec.Count = 1
|
||||
spec.PreferredCount = 5
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 3, target)
|
||||
logger.Infof(msg)
|
||||
nodes = 5
|
||||
spec.PreferredCount = 6
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 5, target)
|
||||
logger.Infof(msg)
|
||||
nodes = 6
|
||||
spec.PreferredCount = 7
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 5, target)
|
||||
logger.Infof(msg)
|
||||
nodes = 7
|
||||
spec.PreferredCount = 7
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 7, target)
|
||||
logger.Infof(msg)
|
||||
|
||||
// multiple allowed per node, so we always returned preferred
|
||||
nodes = 1
|
||||
spec.PreferredCount = 7
|
||||
spec.AllowMultiplePerNode = true
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 7, target)
|
||||
logger.Infof(msg)
|
||||
}
|
||||
|
||||
// mon node usage should return no zones if there are no nodes
|
||||
func TestGetNodeMonUsageNoNodes(t *testing.T) {
|
||||
clientset := test.New(0)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(nodeZones), 0)
|
||||
}
|
||||
|
||||
// all nodes without zone annotations are in the same zone
|
||||
func TestGetNodeMonUsageNoZoneLabels(t *testing.T) {
|
||||
for i := 1; i < 5; i++ {
|
||||
clientset := test.New(i)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(nodeZones), 1)
|
||||
assert.Equal(t, len(nodeZones[0]), i)
|
||||
}
|
||||
}
|
||||
|
||||
// nodes are partitioned into separate zones
|
||||
func TestGetNodeMonUsageZoneSpread(t *testing.T) {
|
||||
clientset := test.New(5)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{})
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion")
|
||||
|
||||
// 1 node labeled -> 1 zone + the rest in the unlabeled zone
|
||||
node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// the zones are unlabeled and sort order isn't guaranteed. so instead we
|
||||
// assert on expected count of zones and collection of zone sizes.
|
||||
zoneCounts := map[int]int{}
|
||||
|
||||
nodeZones, err := c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, len(nodeZones))
|
||||
for _, zones := range nodeZones {
|
||||
zoneSize := len(zones)
|
||||
if _, ok := zoneCounts[zoneSize]; ok {
|
||||
zoneCounts[zoneSize]++
|
||||
} else {
|
||||
zoneCounts[zoneSize] = 1
|
||||
}
|
||||
}
|
||||
assert.Equal(t, map[int]int{1: 1, 4: 1}, zoneCounts)
|
||||
|
||||
// 2 nodes with same label, 3 unlabeled
|
||||
node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// reset
|
||||
zoneCounts = map[int]int{}
|
||||
|
||||
nodeZones, err = c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, len(nodeZones))
|
||||
for _, zones := range nodeZones {
|
||||
zoneSize := len(zones)
|
||||
if _, ok := zoneCounts[zoneSize]; ok {
|
||||
zoneCounts[zoneSize]++
|
||||
} else {
|
||||
zoneCounts[zoneSize] = 1
|
||||
}
|
||||
}
|
||||
assert.Equal(t, map[int]int{2: 1, 3: 1}, zoneCounts)
|
||||
|
||||
// 2 nodes with same label, 2 with distinct labels, 1 unlabeled
|
||||
node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
node, err = clientset.CoreV1().Nodes().Get("node3", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z2"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// reset
|
||||
zoneCounts = map[int]int{}
|
||||
|
||||
nodeZones, err = c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 4, len(nodeZones))
|
||||
for _, zones := range nodeZones {
|
||||
zoneSize := len(zones)
|
||||
if _, ok := zoneCounts[zoneSize]; ok {
|
||||
zoneCounts[zoneSize]++
|
||||
} else {
|
||||
zoneCounts[zoneSize] = 1
|
||||
}
|
||||
}
|
||||
assert.Equal(t, map[int]int{2: 1, 1: 3}, zoneCounts)
|
||||
|
||||
// no unlabeled zones
|
||||
node, err = clientset.CoreV1().Nodes().Get("node4", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, node)
|
||||
node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"}
|
||||
clientset.CoreV1().Nodes().Update(node)
|
||||
|
||||
// reset
|
||||
zoneCounts = map[int]int{}
|
||||
|
||||
nodeZones, err = c.getNodeMonUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, len(nodeZones))
|
||||
for _, zones := range nodeZones {
|
||||
zoneSize := len(zones)
|
||||
if _, ok := zoneCounts[zoneSize]; ok {
|
||||
zoneCounts[zoneSize]++
|
||||
} else {
|
||||
zoneCounts[zoneSize] = 1
|
||||
}
|
||||
}
|
||||
assert.Equal(t, map[int]int{2: 2, 1: 1}, zoneCounts)
|
||||
}
|
||||
@@ -48,9 +48,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) {
|
||||
},
|
||||
}
|
||||
k8sutil.SetOwnerRef(&svcDef.ObjectMeta, &c.ownerRef)
|
||||
if c.Network.IsHost() {
|
||||
svcDef.Spec.ClusterIP = v1.ClusterIPNone
|
||||
}
|
||||
|
||||
// If deploying Nautilus or newer we need a new port for the monitor service
|
||||
if c.ClusterInfo.CephVersion.IsAtLeastNautilus() {
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"path"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rook "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
@@ -52,7 +53,7 @@ func (c *Cluster) getLabels(daemonName string) map[string]string {
|
||||
return labels
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDeployment(monConfig *monConfig, hostname string) *apps.Deployment {
|
||||
func (c *Cluster) makeDeployment(monConfig *monConfig) *apps.Deployment {
|
||||
d := &apps.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: monConfig.ResourceName,
|
||||
@@ -65,7 +66,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, hostname string) *apps.De
|
||||
opspec.AddCephVersionLabelToDeployment(c.ClusterInfo.CephVersion, d)
|
||||
k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef)
|
||||
|
||||
pod := c.makeMonPod(monConfig, hostname)
|
||||
pod := c.makeMonPod(monConfig)
|
||||
replicaCount := int32(1)
|
||||
d.Spec = apps.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
@@ -134,7 +135,49 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig) (*v1.PersistentVolumeClaim, er
|
||||
* Pod spec
|
||||
*/
|
||||
|
||||
func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod {
|
||||
func (c *Cluster) setPodPlacement(pod *v1.PodSpec, p rook.Placement, nodeSelector map[string]string) {
|
||||
p.ApplyToPodSpec(pod)
|
||||
pod.NodeSelector = nodeSelector
|
||||
|
||||
// when a node selector is being used, skip the affinity business below
|
||||
if nodeSelector != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// label selector for monitors used in anti-affinity rules
|
||||
monAntiAffinity := v1.PodAffinityTerm{
|
||||
LabelSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
k8sutil.AppAttr: AppName,
|
||||
},
|
||||
},
|
||||
TopologyKey: v1.LabelHostname,
|
||||
}
|
||||
|
||||
// set monitor pod anti-affinity rules. when monitors should never be
|
||||
// co-located (e.g. not AllowMultiplePerHost or HostNetworking) then the
|
||||
// anti-affinity rule is made to be required during scheduling, otherwise it
|
||||
// is merely a preferred policy.
|
||||
//
|
||||
// ApplyToPodSpec ensures that pod.Affinity is non-nil
|
||||
if pod.Affinity.PodAntiAffinity == nil {
|
||||
pod.Affinity.PodAntiAffinity = &v1.PodAntiAffinity{}
|
||||
}
|
||||
paa := pod.Affinity.PodAntiAffinity
|
||||
|
||||
if c.Network.IsHost() || !c.spec.Mon.AllowMultiplePerNode {
|
||||
paa.RequiredDuringSchedulingIgnoredDuringExecution =
|
||||
append(paa.RequiredDuringSchedulingIgnoredDuringExecution, monAntiAffinity)
|
||||
} else {
|
||||
paa.PreferredDuringSchedulingIgnoredDuringExecution =
|
||||
append(paa.PreferredDuringSchedulingIgnoredDuringExecution, v1.WeightedPodAffinityTerm{
|
||||
Weight: 50,
|
||||
PodAffinityTerm: monAntiAffinity,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMonPod(monConfig *monConfig) *v1.Pod {
|
||||
logger.Debugf("monConfig: %+v", monConfig)
|
||||
podSpec := v1.PodSpec{
|
||||
InitContainers: []v1.Container{
|
||||
@@ -145,7 +188,6 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod {
|
||||
c.makeMonDaemonContainer(monConfig),
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
NodeSelector: map[string]string{v1.LabelHostname: hostname},
|
||||
Volumes: opspec.DaemonVolumesBase(monConfig.DataPathMap, keyringStoreName),
|
||||
HostNetwork: c.Network.IsHost(),
|
||||
}
|
||||
@@ -153,13 +195,6 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
// apply the pod placement if specified in the crd
|
||||
// remove Pod (anti-)affinity because we have our own placement logic
|
||||
p := cephv1.GetMonPlacement(c.spec.Placement)
|
||||
p.PodAffinity = nil
|
||||
p.PodAntiAffinity = nil
|
||||
p.ApplyToPodSpec(&podSpec)
|
||||
|
||||
pod := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: monConfig.ResourceName,
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"testing"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rook "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
|
||||
@@ -64,7 +65,7 @@ func testPodSpec(t *testing.T, monID string, pvc bool) {
|
||||
}
|
||||
monConfig := testGenMonConfig(monID)
|
||||
|
||||
d := c.makeDeployment(monConfig, "node0")
|
||||
d := c.makeDeployment(monConfig)
|
||||
assert.NotNil(t, d)
|
||||
|
||||
if pvc {
|
||||
@@ -145,3 +146,74 @@ func TestDeploymentPVCSpec(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, pvc.Spec.Resources.Requests[v1.ResourceStorage], req)
|
||||
}
|
||||
|
||||
func testPodSpecPlacement(t *testing.T, hostNet, allowMulti bool, req, pref int, placement *rook.Placement) {
|
||||
clientset := testop.New(1)
|
||||
c := New(
|
||||
&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"},
|
||||
"ns",
|
||||
"/var/lib/rook",
|
||||
cephv1.NetworkSpec{HostNetwork: hostNet},
|
||||
metav1.OwnerReference{},
|
||||
&sync.Mutex{},
|
||||
)
|
||||
|
||||
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: allowMulti}, "rook/rook:myversion")
|
||||
monConfig := testGenMonConfig("a")
|
||||
|
||||
if placement != nil {
|
||||
c.spec.Placement = rook.PlacementSpec{}
|
||||
c.spec.Placement["mon"] = *placement
|
||||
}
|
||||
|
||||
d := c.makeDeployment(monConfig)
|
||||
assert.NotNil(t, d)
|
||||
|
||||
p := cephv1.GetMonPlacement(c.spec.Placement)
|
||||
c.setPodPlacement(&d.Spec.Template.Spec, p, nil)
|
||||
|
||||
// should have a required anti-affnity and no preferred anti-affinity
|
||||
assert.Equal(t,
|
||||
req,
|
||||
len(d.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution))
|
||||
assert.Equal(t,
|
||||
pref,
|
||||
len(d.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution))
|
||||
}
|
||||
|
||||
func makePlacement() rook.Placement {
|
||||
return rook.Placement{
|
||||
PodAntiAffinity: &v1.PodAntiAffinity{
|
||||
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
|
||||
{
|
||||
TopologyKey: v1.LabelZoneFailureDomain,
|
||||
},
|
||||
},
|
||||
PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
|
||||
{
|
||||
PodAffinityTerm: v1.PodAffinityTerm{
|
||||
TopologyKey: v1.LabelZoneFailureDomain,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodSpecPlacement(t *testing.T) {
|
||||
// no placement settings in the crd
|
||||
testPodSpecPlacement(t, true, true, 1, 0, nil)
|
||||
testPodSpecPlacement(t, true, false, 1, 0, nil)
|
||||
testPodSpecPlacement(t, false, true, 0, 1, nil)
|
||||
testPodSpecPlacement(t, false, false, 1, 0, nil)
|
||||
|
||||
// crd has other preferred and required anti-affinity setting
|
||||
p := makePlacement()
|
||||
testPodSpecPlacement(t, true, true, 2, 1, &p)
|
||||
p = makePlacement()
|
||||
testPodSpecPlacement(t, true, false, 2, 1, &p)
|
||||
p = makePlacement()
|
||||
testPodSpecPlacement(t, false, true, 1, 2, &p)
|
||||
p = makePlacement()
|
||||
testPodSpecPlacement(t, false, false, 2, 1, &p)
|
||||
}
|
||||
@@ -120,10 +120,6 @@ spec:
|
||||
maximum: 9
|
||||
minimum: 1
|
||||
type: integer
|
||||
preferredCount:
|
||||
maximum: 9
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- count
|
||||
network:
|
||||
|
||||
Reference in new issue
Block a user