From 88e28e7b3ee15f16beb2861f4d72a93c2e699c5b Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Sat, 24 Aug 2019 10:45:48 -0700 Subject: [PATCH 1/3] ceph: fix spurious merge of dead code for some reason 8c8c48e0ac742efc4e1b9c990ce26e0ee18a40a9 introduced some old scheduler code that had previously been removed. Signed-off-by: Noah Watkins --- pkg/operator/ceph/cluster/mon/health.go | 48 --------------- pkg/operator/ceph/cluster/mon/node.go | 77 ------------------------- 2 files changed, 125 deletions(-) diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 1dc60290f..0de6f66b9 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -198,54 +198,6 @@ func (c *Cluster) checkHealth() error { return nil } -func (c *Cluster) checkMonsOnSameNode(desiredMonCount int) (bool, error) { - nodesUsed := map[string]struct{}{} - for name, node := range c.mapping.Node { - // when the node is already in the list we have more than one mon on that node - if _, ok := nodesUsed[node.Name]; ok { - // get list of available nodes for mons - availableNodes, _, err := c.getAvailableMonNodes() - if err != nil { - return true, fmt.Errorf("failed to get available mon nodes. %+v", err) - } - // if there are enough nodes for one mon "that is too much" to be failovered, - // fail it over to an other node - if len(availableNodes) > 0 { - logger.Infof("rebalance: enough nodes available %d to failover mon %s", len(availableNodes), name) - c.failMon(len(c.ClusterInfo.Monitors), desiredMonCount, name) - } else { - logger.Debugf("rebalance: not enough nodes available to failover mon %s", name) - } - - // deal with one mon too much on a node at a time - return true, nil - } - nodesUsed[node.Name] = struct{}{} - } - return false, nil -} - -func (c *Cluster) checkMonsOnValidNodes() (bool, error) { - for mon, nInfo := range c.mapping.Node { - // get node to use for validNode() func - node, err := c.context.Clientset.CoreV1().Nodes().Get(nInfo.Name, metav1.GetOptions{}) - if err != nil { - return true, err - } - // check if node the mon is on is still valid - valid, err := k8sutil.ValidNode(*node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - } else if !valid { - logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon) - c.failoverMon(mon) - return true, nil - } - logger.Debugf("node %s with mon %s is still valid", nInfo.Name, mon) - } - return false, nil -} - // failMon compares the monCount against desiredMonCount func (c *Cluster) failMon(monCount, desiredMonCount int, name string) { if monCount > desiredMonCount { diff --git a/pkg/operator/ceph/cluster/mon/node.go b/pkg/operator/ceph/cluster/mon/node.go index 0ef1744b1..b8c2a74e3 100644 --- a/pkg/operator/ceph/cluster/mon/node.go +++ b/pkg/operator/ceph/cluster/mon/node.go @@ -21,7 +21,6 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/k8sutil" - "github.com/rook/rook/pkg/util" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -37,82 +36,6 @@ type NodeUsage struct { MonValid bool } -// getMonNodes detects the nodes that are available for new mons to start. -func (c *Cluster) getMonNodes() ([]v1.Node, error) { - availableNodes, nodes, err := c.getAvailableMonNodes() - if err != nil { - return nil, err - } - logger.Infof("Found %d running nodes without mons", len(availableNodes)) - - // if all nodes already have mons and the user has given the mon.count, add all nodes to be available - if c.spec.Mon.AllowMultiplePerNode && len(availableNodes) == 0 { - logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items)) - for _, node := range nodes.Items { - valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - } else if valid { - availableNodes = append(availableNodes, node) - } - } - } - - return availableNodes, nil -} - -func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) { - nodeOptions := metav1.ListOptions{} - nodeOptions.TypeMeta.Kind = "Node" - nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions) - if err != nil { - return nil, nil, err - } - logger.Debugf("there are %d nodes available for %d mons", len(nodes.Items), len(c.ClusterInfo.Monitors)) - - // get the nodes that have mons assigned - nodesInUse, err := c.getNodesWithMons(nodes) - if err != nil { - logger.Warningf("could not get nodes with mons. %+v", err) - nodesInUse = util.NewSet() - } - - // choose nodes for the new mons that don't have mons currently - availableNodes := []v1.Node{} - for _, node := range nodes.Items { - if !nodesInUse.Contains(node.Name) { - valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - } else if valid { - availableNodes = append(availableNodes, node) - } - } - } - - return availableNodes, nodes, nil -} - -func (c *Cluster) getNodesWithMons(nodes *v1.NodeList) (*util.Set, error) { - // get the mon pods and their node affinity - options := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)} - pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(options) - if err != nil { - return nil, err - } - nodesInUse := util.NewSet() - for _, pod := range pods.Items { - hostname := pod.Spec.NodeSelector[v1.LabelHostname] - logger.Debugf("mon pod on node %s", hostname) - name, ok := getNodeNameFromHostname(nodes, hostname) - if !ok { - logger.Errorf("mon %s on hostname %s not found in node list", pod.Name, hostname) - } - nodesInUse.Add(name) - } - return nodesInUse, nil -} - // Get the number of mons that the operator should be starting func (c *Cluster) getTargetMonCount() (int, string, error) { From 94bd70a14cdc7ba0b31e441e49c69652f93d12fa Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Thu, 8 Aug 2019 10:06:09 -0700 Subject: [PATCH 2/3] ceph/mon: support monitor migration without rebuils this patch adds two new related features. first, it removes the requirement that all monitor deployments be pinned to nodes using node selector. this is allowed when not using host networking and when PVC storage is used. the second feature introduced in this patch is the elimination of the Rook scheduler for placing monitors that continue to use the node selector by using canary pods to that are scheduled by k8s, after which Rook uses the scheduling results to pin the monitors to nodes. fixes: #3667 Signed-off-by: Noah Watkins --- cluster/examples/kubernetes/ceph/cluster.yaml | 4 + design/ceph-mon-pv.md | 53 ++- pkg/operator/ceph/cluster/mon/config.go | 2 - pkg/operator/ceph/cluster/mon/health.go | 182 +-------- pkg/operator/ceph/cluster/mon/health_test.go | 308 +-------------- pkg/operator/ceph/cluster/mon/mon.go | 369 ++++++++++++------ pkg/operator/ceph/cluster/mon/mon_test.go | 171 +------- pkg/operator/ceph/cluster/mon/node.go | 73 ---- pkg/operator/ceph/cluster/mon/node_test.go | 272 +------------ pkg/operator/ceph/cluster/mon/service.go | 3 - pkg/operator/ceph/cluster/mon/spec.go | 57 ++- pkg/operator/ceph/cluster/mon/spec_test.go | 74 +++- 12 files changed, 450 insertions(+), 1118 deletions(-) diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 06085dc6c..221b71a3d 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -79,6 +79,10 @@ spec: # operator: Exists # The above placement information can also be specified for mon, osd, and mgr components # mon: +# Monitor deployments may contain an anti-affinity rule for avoiding monitor +# colocation on the same node. This is a required rule when host network is used +# or when AllowMultiplePerNode is false. Otherwise this anti-affinity rule is a +# preferred rule with weight: 50. # osd: # mgr: annotations: diff --git a/design/ceph-mon-pv.md b/design/ceph-mon-pv.md index 4d04dbccb..9a513deaf 100644 --- a/design/ceph-mon-pv.md +++ b/design/ceph-mon-pv.md @@ -89,9 +89,52 @@ scheduling implementation in Rook. These restrictions will be removed or significantly relaxed once monitor scheduling is moved under control of Kubernetes itself (ongoing work). -## Migration +## Scheduling -Monitor migration is currently not implemented. When nodes are decommissioned, -for example, any monitors on the nodes are migrated using a fail over mechanism -rather than also migrating the PVC. This support is currently being worked on in -conjunction with the enhancements to scheduling. +In previous versions of Rook the operator made explicit scheduling (placement) +decisions when creating monitor deployments. These decisions were made by +implementing a custom scheduling algorithm, and using the pod node selector to +enforce the placement decision. Unfortunately, schedulers are difficult to +write correctly, and manage. Furthermore, by maintaining a separate scheduler +from Kubernetes global policies are difficult to achieve. + +Despite the benefits of using the Kubernetes scheduler, there are important use +cases for using a node selector on a monitor deployment: pinning a monitor to a +node when HostPath-based storage is used. In this case Rook must prevent k8s +from moving a pod away from the node that contains its storage. The node +selector is used to enforce this affinity. Unfortunately, node selector use is +mutually exclusive with kubernetes scheduling---a pod cannot be scheduled by +Kubernetes and then atomically have its affinity set to that placement decision. + +The workaround in Rook is to use a temporary _canary pod_ that is scheduled by +Kubernetes, but whose placement is enforced by Rook. The canary deployment is a +deployment configured identically to a monitor deployment, except the container +entrypoints have no side affects. The canary deployments are used to solve a +fundamental bootstrapping issue: we want to avoid making explicit scheduling +decisions in Rook, but in some configurations a node selector needs to be used +to pin a monitor to a node. + +### Health checks + +Previous versions of Rook performed periodic health checks that included checks +on monitor health as well as looking for scheduling violations. The health +checks related to scheduling violations have been removed. Fundamentally a +placement violation requires understanding or accessing the scheduling algorithm. + +The rescheduling or eviction aspects of Rook's scheduling caused more problems +than it helped, so going with K8s scheduling is the right thing. If/when K8s +has eviction policies in the future we could then make use of it (e.g. with +`*RequiredDuringExecution` variants of anti-affinity rules are available). + +### Target Monitor Count + +The CRD monitor count specifies a target minimum number of monitors to maintain. +Additionally, a preferred count is available which will be the desired number of +sufficient number of nodes are available. Unfortunately, this calculation +relies on determining if monitor pods may be placed on a node, requiring +knowledge of the scheduling policy and algorithm. The scenario to watch out for +is an endless loop in which the health check is determining a bad placement but +the k8s schedule thinks otherwise. + +**TODO**: one approach here may be to schedule the desired number of canary pods +to check for capacity before proceeding through monitor orchestration. diff --git a/pkg/operator/ceph/cluster/mon/config.go b/pkg/operator/ceph/cluster/mon/config.go index f62a411b9..d02a8523c 100644 --- a/pkg/operator/ceph/cluster/mon/config.go +++ b/pkg/operator/ceph/cluster/mon/config.go @@ -81,7 +81,6 @@ func CreateOrLoadClusterInfo(context *clusterd.Context, namespace string, ownerR maxMonID := -1 monMapping := &Mapping{ Node: map[string]*NodeInfo{}, - Port: map[string]int32{}, } secrets, err := context.Clientset.CoreV1().Secrets(namespace).Get(AppName, metav1.GetOptions{}) @@ -137,7 +136,6 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string maxMonID := -1 monMapping := &Mapping{ Node: map[string]*NodeInfo{}, - Port: map[string]int32{}, } cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 0de6f66b9..d08f2a55f 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -25,11 +25,8 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" cephutil "github.com/rook/rook/pkg/daemon/ceph/util" - "github.com/rook/rook/pkg/operator/k8sutil" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - scheduler "k8s.io/kubernetes/pkg/scheduler/api" ) var ( @@ -172,13 +169,6 @@ func (c *Cluster) checkHealth() error { return nil } - // find any mons that invalidate our placement policy, and if necessary, - // reschedule them to other nodes. - done, err := c.resolveInvalidMonitorPlacement(desiredMonCount) - if done || err != nil { - return err - } - // create/start new mons when there are fewer mons than the desired count in the CRD if len(status.MonMap.Mons) < desiredMonCount { logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(status.MonMap.Mons), desiredMonCount) @@ -220,16 +210,10 @@ func (c *Cluster) failoverMon(name string) error { m := c.newMonConfig(c.maxMonID + 1) logger.Infof("starting new mon: %+v", m) - // Create the service endpoint - serviceIP, err := c.createService(m) - if err != nil { - return fmt.Errorf("failed to create mon service. %+v", err) - } - mConf := []*monConfig{m} // Assign the pod to a node - if err = c.assignMons(mConf); err != nil { + if err := c.assignMons(mConf); err != nil { return fmt.Errorf("failed to place new mon on a node. %+v", err) } @@ -240,12 +224,17 @@ func (c *Cluster) failoverMon(name string) error { } m.PublicIP = node.Address } else { + // Create the service endpoint + serviceIP, err := c.createService(m) + if err != nil { + return fmt.Errorf("failed to create mon service. %+v", err) + } m.PublicIP = serviceIP } c.ClusterInfo.Monitors[m.DaemonName] = cephconfig.NewMonInfo(m.DaemonName, m.PublicIP, m.Port) // Start the deployment - if err = c.startDeployments(mConf, true); err != nil { + if err := c.startDeployments(mConf, true); err != nil { return fmt.Errorf("failed to start new mon %s. %+v", m.DaemonName, err) } @@ -279,19 +268,7 @@ func (c *Cluster) removeMon(daemonName string) error { delete(c.ClusterInfo.Monitors, daemonName) // check if a mapping exists for the mon if _, ok := c.mapping.Node[daemonName]; ok { - nodeName := c.mapping.Node[daemonName].Name delete(c.mapping.Node, daemonName) - // if node->port "mapping" has been created, decrease or delete it - if port, ok := c.mapping.Port[nodeName]; ok { - if port == DefaultMsgr1Port { - delete(c.mapping.Port, nodeName) - } - // don't clean up if a node port is higher than the default port, other - // mons could be on the same node with > DefaultPort ports, decreasing could - // cause port collisions - // This can be solved by using a map[nodeName][]int32 for the ports to - // even better check which ports are open for the HostNetwork mode - } } // Remove the service endpoint @@ -321,151 +298,6 @@ func removeMonitorFromQuorum(context *clusterd.Context, clusterName, name string return nil } -func (c *Cluster) resolveInvalidMonitorPlacement(desiredMonCount int) (bool, error) { - nodeChoice, err := c.findInvalidMonitorPlacement(desiredMonCount) - if err != nil { - return true, fmt.Errorf("failed to find invalid mon placement %+v", err) - } - - // no violation found - if nodeChoice == nil { - return false, nil - } - - for name, nodeInfo := range c.mapping.Node { - if nodeInfo.Name == nodeChoice.Node.Name { - logger.Infof("rebalance: rescheduling mon %s from node %s", name, nodeInfo.Name) - c.failMon(len(c.ClusterInfo.Monitors), desiredMonCount, name) - return true, nil - } - } - - logger.Warningf("rebalance: no mon pod found on node %s", nodeChoice.Node.Name) - - return false, nil -} - -func (c *Cluster) findInvalidMonitorPlacement(desiredMonCount int) (*NodeUsage, error) { - nodeZones, err := c.getNodeMonUsage() - if err != nil { - return nil, fmt.Errorf("failed to get node monitor usage. %+v", err) - } - - // compute two helpful global flags: - // - does an empty zone exist - // - does an empty node exist - emptyZone := false - emptyNode := false - for zi := range nodeZones { - monFoundInZone := false - for ni := range nodeZones[zi] { - nodeUsage := &nodeZones[zi][ni] - if nodeUsage.MonCount > 0 { - monFoundInZone = true - } else if nodeUsage.MonValid { - // only consider valid nodes since this flag determines if a mon - // may be scheduled on to a new node. - emptyNode = true - } - } - if !monFoundInZone { - // emptyZone is used below to imply that an empty zone exists AND - // that zone contains a valid node (i.e. a node that new pods can be - // assigned). this check enforces that assumption by checking that - // the zone contains at least one valid node. - validNodeInZone := false - for _, nodeUsage := range nodeZones[zi] { - if nodeUsage.MonValid { - validNodeInZone = true - break - } - } - if validNodeInZone { - emptyZone = true - } - logger.Debugf("rebalance: empty zone found. validNodeInZone: %t, emptyZone %t", - validNodeInZone, emptyZone) - } - } - - logger.Debugf("rebalance: desired mon count: %d, empty zone found: %t, empty node found: %t", - desiredMonCount, emptyZone, emptyNode) - - for zi := range nodeZones { - zoneMonCount := 0 - var nodeChoice *NodeUsage - - // for each node consider two cases: too many monitors are running on a - // node, or monitors are running on invalid nodes. - for ni := range nodeZones[zi] { - nodeUsage := &nodeZones[zi][ni] - zoneMonCount += nodeUsage.MonCount - - // if this node has too many monitors, and an underused node exists, - // then consider moving the monitor. note that this check is - // independent of the setting `AllowMultiplePerNode` since we also - // want to avoid in general keeping multiple monitors on one node. - if nodeUsage.MonCount > 1 && emptyNode { - logger.Infof("rebalance: chose overloaded node %s with %d mons", - nodeUsage.Node.Name, nodeUsage.MonCount) - nodeChoice = nodeUsage - break - } - - // check for mons on invalid nodes. but reschedule pod only when it - // is invalid for reasons other than schedulability which should not - // affect pods that are already scheduled. - if nodeUsage.MonCount > 0 && !nodeUsage.MonValid { - placement := cephv1.GetMonPlacement(c.spec.Placement) - placement.Tolerations = append(placement.Tolerations, - v1.Toleration{ - Key: scheduler.TaintNodeUnschedulable, - Effect: "", - Operator: v1.TolerationOpExists, - }) - valid, err := k8sutil.ValidNodeNoSched(*nodeUsage.Node, placement) - if err != nil { - logger.Warningf("rebalance: failed to validate node %s %+v", - nodeUsage.Node.Name, err) - } else if !valid { - logger.Infof("rebalance: chose invalid node %s", - nodeUsage.Node.Name) - nodeChoice = nodeUsage - break - } - } - } - - // if we didn't already select a node with a monitor for rescheduling, - // then consider that a zone may be overloaded and can be rebalanced. - // this case occurs if the zone has more than 1 monitor, and some other - // zone exists without any monitors. - if nodeChoice == nil && zoneMonCount > 1 && emptyZone { - for ni := range nodeZones[zi] { - nodeUsage := &nodeZones[zi][ni] - if nodeUsage.MonCount > 0 { - // select the node with the most monitors assigned - if nodeChoice == nil || nodeUsage.MonCount > nodeChoice.MonCount { - nodeChoice = nodeUsage - } - } - } - if nodeChoice != nil { - logger.Infof("rebalance: chose node %s from overloaded zone", - nodeChoice.Node.Name) - } - } - - if nodeChoice != nil { - return nodeChoice, nil - } - } - - logger.Debugf("rebalance: no mon placement violations or fixes available") - - return nil, nil -} - func (c *Cluster) handleExternalMonStatus(status client.MonStatusResponse) error { changed, err := c.addOrRemoveExternalMonitor(status) diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index c4264f9e7..3c059d263 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -17,7 +17,6 @@ limitations under the License. package mon import ( - "encoding/json" "fmt" "io/ioutil" "os" @@ -34,7 +33,6 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -66,9 +64,14 @@ func TestCheckHealth(t *testing.T) { Name: "node0", Address: "", } - c.mapping.Port["node0"] = DefaultMsgr1Port c.maxMonID = 4 + // mock out the scheduler to return node0 + scheduleMonitor = func(c *Cluster, mon *monConfig) (SchedulingResult, error) { + node, _ := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + return SchedulingResult{Node: node}, nil + } + err := c.checkHealth() assert.Nil(t, err) logger.Infof("mons after checkHealth: %v", c.ClusterInfo.Monitors) @@ -117,7 +120,6 @@ func TestCheckHealthNotFound(t *testing.T) { c.mapping.Node["b"] = &NodeInfo{ Name: "node0", } - c.mapping.Port["node0"] = DefaultMsgr1Port c.maxMonID = 4 c.saveMonConfig() @@ -145,304 +147,6 @@ func TestCheckHealthNotFound(t *testing.T) { } } -func TestOverloadedZoneRebalances(t *testing.T) { - clientset := test.New(3) - - // build a cluster with 3 nodes. two of the nodes are in the same AZ the - // third node is in a separate AZ. - for i := 0; i < 3; i++ { - nodeName := fmt.Sprintf("node%d", i) - node, err := clientset.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - - // zone0: node0, node1 - // zone1: node2 - var zoneName string - if i < 2 { - zoneName = "zone0" - } else { - zoneName = "zone1" - } - - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": zoneName} - clientset.CoreV1().Nodes().Update(node) - } - - executor := &exectest.MockExecutor{} - - configDir, _ := ioutil.TempDir("", "") - defer os.RemoveAll(configDir) - context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - } - - cluster := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(cluster, 2, cephv1.MonSpec{Count: 2, AllowMultiplePerNode: false}, "myversion") - cluster.waitForStart = false - defer os.RemoveAll(cluster.context.ConfigDir) - - // add mon mappings - cluster.mapping.Node["a"] = &NodeInfo{ - Name: "node0", - Hostname: "mynode0", - Address: "0.0.0.0", - } - cluster.mapping.Node["b"] = &NodeInfo{ - Name: "node1", - Hostname: "mynode1", - Address: "0.0.0.0", - } - cluster.maxMonID = 1 - cluster.saveMonConfig() - - monNames := []string{"a", "b"} - nodeNames := []string{"node0", "node1"} - for i := 0; i < len(monNames); i++ { - monConfig := testGenMonConfig(monNames[i]) - nodeName := nodeNames[i] - - d := cluster.makeDeployment(monConfig, nodeName) - _, err := clientset.AppsV1().Deployments(cluster.Namespace).Create(d) - assert.Nil(t, err) - - po := cluster.makeMonPod(monConfig, nodeName) - _, err = clientset.CoreV1().Pods(cluster.Namespace).Create(po) - assert.Nil(t, err) - } - - assert.NotContains(t, cluster.mapping.Node, "c") - assert.Equal(t, "node0", cluster.mapping.Node["a"].Name) - assert.Equal(t, "node1", cluster.mapping.Node["b"].Name) - assert.Equal(t, "mynode0", cluster.mapping.Node["a"].Hostname) - assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname) - - _, err := cluster.resolveInvalidMonitorPlacement(2) - assert.Nil(t, err) - - // mon.a was chosen from the overloaded zone 0, and a new monitor to take - // its place, mon.c was created on a node in zone 1 that had no monitors. - assert.NotContains(t, cluster.mapping.Node, "a") - assert.Equal(t, "node1", cluster.mapping.Node["b"].Name) - assert.Equal(t, "node2", cluster.mapping.Node["c"].Name) - assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname) -} - -func TestCheckHealthTwoMonsOneNode(t *testing.T) { - executorNextMons := false - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { - if executorNextMons { - resp := client.MonStatusResponse{Quorum: []int{0}} - resp.MonMap.Mons = []client.MonMapEntry{ - { - Name: "a", - Rank: 0, - Address: "1.2.3.4", - }, - { - Name: "c", - Rank: 0, - Address: "1.2.3.4", - }, - } - serialized, _ := json.Marshal(resp) - return string(serialized), nil - } - return clienttest.MonInQuorumResponseMany(2), nil - }, - } - clientset := test.New(1) - configDir, _ := ioutil.TempDir("", "") - defer os.RemoveAll(configDir) - context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - } - - c := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 2, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - c.waitForStart = false - defer os.RemoveAll(c.context.ConfigDir) - - // add two mons to the mapping on node0 - c.mapping.Node["a"] = &NodeInfo{ - Name: "node0", - Hostname: "mynode0", - Address: "0.0.0.0", - } - c.mapping.Node["b"] = &NodeInfo{ - Name: "node0", - Hostname: "mynode0", - Address: "0.0.0.0", - } - c.maxMonID = 1 - c.saveMonConfig() - - monNames := []string{"a", "b"} - for i := 0; i < len(monNames); i++ { - monConfig := testGenMonConfig(monNames[i]) - d := c.makeDeployment(monConfig, "node0") - _, err := clientset.AppsV1().Deployments(c.Namespace).Create(d) - assert.Nil(t, err) - po := c.makeMonPod(monConfig, "node0") - _, err = clientset.CoreV1().Pods(c.Namespace).Create(po) - assert.Nil(t, err) - } - - // initial health check should already see that there is more than one mon on one node (node0) - _, err := c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - assert.Equal(t, "node0", c.mapping.Node["a"].Name) - assert.Equal(t, "node0", c.mapping.Node["b"].Name) - assert.Equal(t, "mynode0", c.mapping.Node["a"].Hostname) - assert.Equal(t, "mynode0", c.mapping.Node["b"].Hostname) - - // add new node and check if the second mon gets failovered to it - n := &v1.Node{ - Status: v1.NodeStatus{ - Conditions: []v1.NodeCondition{ - { - Type: v1.NodeReady, Status: v1.ConditionTrue, - }, - }, - Addresses: []v1.NodeAddress{ - { - Type: v1.NodeExternalIP, - Address: "2.2.2.2", - }, - }, - }, - } - n.Name = "node2" - clientset.CoreV1().Nodes().Create(n) - - _, err = c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - - // check that mon c exists - logger.Infof("mapping: %+v", c.mapping.Node) - assert.NotNil(t, c.mapping.Node["c"]) - assert.Equal(t, "node2", c.mapping.Node["c"].Name) - - // check if mon b has been deleted - var dlist *apps.DeploymentList - dlist, err = clientset.AppsV1().Deployments(c.Namespace).List(metav1.ListOptions{}) - assert.Nil(t, err) - deleted := true - for _, d := range dlist.Items { - if d.Name == "b" { - deleted = false - } - } - assert.Equal(t, true, deleted, "mon b not failed over or deleted after health check") - - // enable different ceph mon map output - executorNextMons = true - _, err = c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - - // check that nothing has changed - dlist, err = clientset.AppsV1().Deployments(c.Namespace).List(metav1.ListOptions{}) - assert.Nil(t, err) - - for _, d := range dlist.Items { - // both mons should always be on the same node as in this test due to the order - // the mons are processed in the loop - if (d.Name == "rook-ceph-mon-a" && d.Spec.Template.Spec.NodeSelector[v1.LabelHostname] == "node1") || (d.Name != "rook-ceph-mon-c" && d.Spec.Template.Spec.NodeSelector[v1.LabelHostname] == "node2") { - assert.Fail(t, fmt.Sprintf("mon %s shouldn't exist", d.Name)) - } - } -} - -func TestCheckMonsValid(t *testing.T) { - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { - return clienttest.MonInQuorumResponse(), nil - }, - } - clientset := test.New(1) - configDir, _ := ioutil.TempDir("", "") - defer os.RemoveAll(configDir) - context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - } - c := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - c.waitForStart = false - defer os.RemoveAll(c.context.ConfigDir) - - // add two mons to the mapping on node0 - c.mapping.Node["a"] = &NodeInfo{ - Name: "node0", - Address: "0.0.0.0", - } - c.mapping.Node["b"] = &NodeInfo{ - Name: "node1", - Address: "0.0.0.0", - } - c.maxMonID = 1 - - // add three nodes - for i := 0; i < 3; i++ { - n := &v1.Node{ - Status: v1.NodeStatus{ - Conditions: []v1.NodeCondition{ - { - Type: v1.NodeReady, Status: v1.ConditionTrue, - }, - }, - Addresses: []v1.NodeAddress{ - { - Type: v1.NodeExternalIP, - Address: "0.0.0.0", - }, - }, - }, - } - n.Name = fmt.Sprintf("node%d", i) - clientset.CoreV1().Nodes().Create(n) - } - - _, err := c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - assert.Equal(t, "node0", c.mapping.Node["a"].Name) - assert.Equal(t, "node1", c.mapping.Node["b"].Name) - - // set node0 to contain the not ready status condition, and check that mon.a - // is failovered to be mon.c on node2. note that validity here doesn't include the - // unscheduable flag (for mons already placed on a node). - node0, err := c.context.Clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.Nil(t, err) - node0.Status.Conditions[0].Status = v1.ConditionFalse - _, err = c.context.Clientset.CoreV1().Nodes().Update(node0) - assert.Nil(t, err) - - // add the pods so the getNodesInUse() works correctly - monNames := []string{"a", "b", "c"} - for i := 1; i <= 2; i++ { // 1=b, 2=c - po := c.makeMonPod(testGenMonConfig(monNames[i]), fmt.Sprintf("node%d", i-1)) - _, err = clientset.CoreV1().Pods(c.Namespace).Create(po) - assert.Nil(t, err) - } - - _, err = c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - - assert.Len(t, c.mapping.Node, 2) - logger.Infof("mapping: %+v", c.mapping) - assert.Nil(t, c.mapping.Node["a"]) - // the new mon should always be on the empty node2 - // the failovered mon's name is "rook-ceph-mon-a" - assert.Equal(t, "node2", c.mapping.Node["c"].Name) - assert.Equal(t, "node1", c.mapping.Node["b"].Name) -} - func TestAddRemoveMons(t *testing.T) { var deploymentsUpdated *[]*apps.Deployment updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub() diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index c9df8f58a..e0099ac1b 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -43,6 +43,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) const ( @@ -83,10 +84,18 @@ const ( // default storage request size for ceph monitor pvc // https://docs.ceph.com/docs/master/start/hardware-recommendations/#monitors-and-managers-ceph-mon-and-ceph-mgr cephMonDefaultStorageRequest = "10Gi" + + // canary pod scheduling uses retry loops when cleaning up previous canary + // pods and waiting for kubernetes scheduling to complete. + canaryRetries = 180 + canaryRetryDelaySeconds = 5 ) var ( logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mon") + + // hook for tests to override + scheduleMonitor = realScheduleMonitor ) // Cluster represents the Rook and environment configuration settings needed to set up Ceph mons. @@ -129,7 +138,6 @@ type monConfig struct { // Mapping is mon node and port mapping type Mapping struct { Node map[string]*NodeInfo `json:"node"` - Port map[string]int32 `json:"port"` } // NodeInfo contains name and address of a node @@ -139,6 +147,12 @@ type NodeInfo struct { Address string } +type SchedulingResult struct { + Node *v1.Node + CanaryDeployment string + CanaryPVC string +} + // New creates an instance of a mon cluster func New(context *clusterd.Context, namespace, dataDirHostPath string, network cephv1.NetworkSpec, ownerRef metav1.OwnerReference, csiConfigMutex *sync.Mutex) *Cluster { return &Cluster{ @@ -153,7 +167,6 @@ func New(context *clusterd.Context, namespace, dataDirHostPath string, network c Network: network, mapping: &Mapping{ Node: map[string]*NodeInfo{}, - Port: map[string]int32{}, }, ownerRef: ownerRef, csiConfigMutex: csiConfigMutex, @@ -365,6 +378,136 @@ func resourceName(name string) string { return fmt.Sprintf("%s-%s", AppName, name) } +// scheduleMonitor selects a node for a monitor deployment. +// see startMon() and design/ceph-mon-pv.md for additional details. +func realScheduleMonitor(c *Cluster, mon *monConfig) (SchedulingResult, error) { + // target node decision, and deployment/pvc to cleanup + result := SchedulingResult{ + Node: nil, + CanaryDeployment: "", + CanaryPVC: "", + } + + // build the canary deployment. + d := c.makeDeployment(mon) + d.Name += "-canary" + d.Spec.Template.ObjectMeta.Name += "-canary" + + // the canary and real monitor deployments will mount the same storage. to + // avoid issues with the real deployment, the canary should be careful not + // to modify the storage by instead running an innocuous command. + d.Spec.Template.Spec.InitContainers = []v1.Container{} + d.Spec.Template.Spec.Containers[0].Image = c.rookVersion + d.Spec.Template.Spec.Containers[0].Command = []string{"/tini"} + d.Spec.Template.Spec.Containers[0].Args = []string{"--", "sleep", "3600"} + + // setup affinity settings for pod scheduling + p := cephv1.GetMonPlacement(c.spec.Placement) + c.setPodPlacement(&d.Spec.Template.Spec, p, nil) + + // setup storage on the canary since scheduling will be affected when + // monitors are configured to use persistent volumes. the pvcName is set to + // the non-empty name of the PVC only when the PVC is created as a result of + // this call to the scheduler. + var pvcName string + if c.spec.Mon.VolumeClaimTemplate == nil { + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, + opspec.DaemonVolumesDataHostPath(mon.DataPathMap)...) + } else { + // the pvc that is created here won't be deleted: it will be reattached + // to the real monitor deployment. + pvc, err := c.makeDeploymentPVC(mon) + if err != nil { + return result, fmt.Errorf("sched-mon: failed to make monitor %s pvc. %+v", d.Name, err) + } + + _, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(pvc) + if err == nil { + pvcName = pvc.Name + logger.Infof("sched-mon: created canary monitor %s pvc %s", d.Name, pvc.Name) + } else { + if errors.IsAlreadyExists(err) { + logger.Debugf("sched-mon: creating mon %s pvc %s: already exists.", d.Name, pvc.Name) + } else { + return result, fmt.Errorf("sched-mon: error creating mon %s pvc %s. %+v", d.Name, pvc.Name, err) + } + } + + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, + opspec.DaemonVolumesDataPVC(mon.ResourceName)) + opspec.AddVolumeMountSubPath(&d.Spec.Template.Spec, "ceph-daemon-data") + } + + // caller should arrange for clean-up of the PVC only if it was created + // during this scheduling instance and we encounter an irrecoverable error. + result.CanaryPVC = pvcName + + // spin up the canary deployment. if it exists, delete it first, since if it + // already exists it may have been scheduled with a different crd config. + createdDeployment := false + for i := 0; i < canaryRetries; i++ { + _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d) + if err == nil { + createdDeployment = true + logger.Infof("sched-mon: created canary deployment %s", d.Name) + break + } else if errors.IsAlreadyExists(err) { + if err := k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, d.Name); err != nil { + return result, fmt.Errorf("sched-mon: error deleting canary deployment %s. %+v", d.Name, err) + } + logger.Infof("sched-mon: deleted existing canary deployment %s", d.Name) + time.Sleep(time.Second * canaryRetryDelaySeconds) + } else { + return result, fmt.Errorf("sched-mon: error creating canary monitor deployment %s. %+v", d.Name, err) + } + } + + // failed after retrying + if !createdDeployment { + return result, fmt.Errorf("sched-mon: failed to create canary deployment %s", d.Name) + } + + // caller should arrange for the deployment to be removed + result.CanaryDeployment = d.Name + + // wait for the scheduler to make a placement decision + for i := 0; i < canaryRetries; i++ { + listOptions := metav1.ListOptions{ + LabelSelector: labels.Set(d.Spec.Selector.MatchLabels).String(), + } + + pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(listOptions) + if err != nil { + return result, fmt.Errorf("sched-mon: error listing canary pods %s. %+v", d.Name, err) + } + + if len(pods.Items) == 0 { + logger.Infof("sched-mon: waiting for canary pod creation %s", d.Name) + time.Sleep(time.Second * canaryRetryDelaySeconds) + continue + } + + pod := pods.Items[0] + if pod.Spec.NodeName == "" { + logger.Debugf("sched-mon: monitor %s canary pod %s not yet scheduled", d.Name, pod.Name) + time.Sleep(time.Second * canaryRetryDelaySeconds) + continue + } + + node, err := c.context.Clientset.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + return result, fmt.Errorf("sched-mon: error getting node %s. %+v", pod.Spec.NodeName, err) + } + + result.Node = node + result.CanaryPVC = "" + logger.Infof("sched-mon: canary monitor deployment %s scheduled to %s", d.Name, node.Name) + return result, nil + } + + return result, fmt.Errorf("sched-mon: canary pod scheduling failed retries") +} + func (c *Cluster) initMonIPs(mons []*monConfig) error { for _, m := range mons { if c.Network.IsHost() { @@ -387,129 +530,82 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error { return nil } -// find a suitable node for this monitor. first look for a fault zone that has -// no monitors. only after the nodes with zone labels are considered are the -// unlabeled nodes considered. reasonsing: without more information, _at worst_ -// the unlabled nodes are actually in the set of labeled zones, and at best, -// they are in distinct zones. -func scheduleMonitor(mon *monConfig, nodeZones [][]NodeUsage) *NodeUsage { - // the node choice for this monitor - var nodeChoice *NodeUsage - - // for each zone, in order of preference. unlabeled nodes are last, by - // construction; see Cluster.getNodeMonusage(). - for zi := range nodeZones { - // number of monitors in the zone - zoneMonCount := 0 - - // preferential node choice from this zone for a new mon - var zoneNodeChoice *NodeUsage - - // for each node in this zone - for ni := range nodeZones[zi] { - nodeUsage := &nodeZones[zi][ni] - - // skip invalid nodes. but update the mon count first since - // invalid nodes could still have an assigned mon pod. - zoneMonCount += nodeUsage.MonCount - if !nodeUsage.MonValid { - logger.Infof("schedmon: skip invalid node %s for mon scheduling", - nodeUsage.Node.Name) - continue +func (c *Cluster) assignMons(mons []*monConfig) error { + // when monitors are scheduling below by invoking scheduleMonitor() a canary + // deployment and optional canary PVC are created. in order for the + // anti-affinity rules to be effective, we leave the canary pods in place + // until all of the canaries have been scheduled. only after the + // monitor/node assignment process is complete are the canary deployments + // and pvcs removed here. + canaryCleanup := []SchedulingResult{} + defer func() { + for _, result := range canaryCleanup { + logger.Infof("assignmon: cleaning up canary deployment %s and canary pvc %s", result.CanaryDeployment, result.CanaryPVC) + if result.CanaryDeployment != "" { + if err := k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, result.CanaryDeployment); err != nil { + logger.Infof("assignmon: error deleting canary deployment %s. %+v", result.CanaryDeployment, err) + } } - - // make a "best" choice from this zone. in this case that is the - // node with the least amount of monitors. - if zoneNodeChoice == nil || nodeUsage.MonCount < zoneNodeChoice.MonCount { - logger.Infof("schedmon: considering node %s with mon count %d", - nodeUsage.Node.Name, nodeUsage.MonCount) - zoneNodeChoice = nodeUsage - } - } - - // We identified _a_ valid node in the zone. should we choose it? - if zoneNodeChoice != nil { - if zoneMonCount == 0 { - // this zone has no monitors, which implies that - // zoneNodeChoice will be a node without any monitors - // currently assigned. choose this node. - logger.Infof("schedmon: considering node %s from empty zone", - zoneNodeChoice.Node.Name) - nodeChoice = zoneNodeChoice - break - } else { - // this zone already has a monitor in it. but keep the - // choice as a backup; we may find that all zones already - // have a monitor and still need to make an assignment. - if nodeChoice == nil || zoneNodeChoice.MonCount < nodeChoice.MonCount { - logger.Infof("schedmon: considering node %s with mon count %d", - zoneNodeChoice.Node.Name, zoneNodeChoice.MonCount) - nodeChoice = zoneNodeChoice + if result.CanaryPVC != "" { + var gracePeriod int64 // delete immediately + propagation := metav1.DeletePropagationForeground + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(result.CanaryPVC, options) + if err != nil { + logger.Infof("assignmon: error removing canary monitor %s pvc %s. %+v", result.CanaryDeployment, result.CanaryPVC, err) } } } - } + }() - if nodeChoice != nil { - logger.Infof("schedmon: scheduling mon %s on node %s", - mon.DaemonName, nodeChoice.Node.Name) - } else { - logger.Infof("schedmon: no suitable node found for mon %s", - mon.DaemonName) - } - - return nodeChoice -} - -func (c *Cluster) assignMons(mons []*monConfig) error { - - // retrieve the set of cluster nodes and their monitor usage info - nodeZones, err := c.getNodeMonUsage() - if err != nil { - return fmt.Errorf("failed to get node monitor usage. %+v", err) - } - - // ensure all monitors have a node assignment. note that this isn't - // necessarily optimal: it does not try to move existing monitors which is - // handled by the periodic monitor health checks. + // ensure that all monitors have either (1) a node assignment that will be + // enforced using a node selector, or (2) configuration permits k8s to handle + // scheduling for the monitor. for _, mon := range mons { - // monitor is already assigned to a node. nothing to do + // scheduling for this monitor has already been completed if _, ok := c.mapping.Node[mon.DaemonName]; ok { - logger.Infof("assignmon: mon %s already assigned to a node", mon.DaemonName) + logger.Debugf("assignmon: mon %s already scheduled", mon.DaemonName) continue } - nodeChoice := scheduleMonitor(mon, nodeZones) + // determine a placement for the monitor. note that this scheduling is + // performed even when a node selector is not required. this may be + // non-optimal, but it is convenient to catch some failures early, + // before a decision is stored in the node mapping. + result, err := scheduleMonitor(c, mon) - if nodeChoice == nil { - return fmt.Errorf("assignmon: no valid nodes available for mon placement") - } + // even if an error occurs cleanup may be necessary + canaryCleanup = append(canaryCleanup, result) - // note that we do not need to worry about a false negative here (i.e. a - // node exists that _would_ pass this check) due to the search landing in a - // local minima because if a node had existed with a mon count of zero, - // scheduleMonitor would have chosen it over any node with a positive - // mon count. - if nodeChoice.MonCount > 0 && !c.spec.Mon.AllowMultiplePerNode { - return fmt.Errorf("assignmon: no empty nodes available for mon placement") - } - - // make this decision visible when scheduling the next monitor - nodeChoice.MonCount++ - - logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Node.Name) - - nodeInfo, err := getNodeInfoFromNode(*nodeChoice.Node) if err != nil { - return fmt.Errorf("couldn't get node info from node %s. %+v", - nodeChoice.Node.Name, err) + return fmt.Errorf("assignmon: error scheduling monitor: %+v", err) + } + + nodeChoice := result.Node + if nodeChoice == nil { + return fmt.Errorf("assignmon: could not schedule monitor %s", mon.DaemonName) + } + + // store nil in the node mapping to indicate that an explicit node + // placement is not being made. otherwise, the node choice will map + // directly to a node selector on the monitor pod. + var nodeInfo *NodeInfo = nil + if c.Network.IsHost() || c.spec.Mon.VolumeClaimTemplate == nil { + logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name) + nodeInfo, err = getNodeInfoFromNode(*nodeChoice) + if err != nil { + return fmt.Errorf("assignmon: couldn't get node info for node %s. %+v", + nodeChoice.Name, err) + } + } else { + logger.Infof("assignmon: mon %s placement using native scheduler", mon.DaemonName) } c.mapping.Node[mon.DaemonName] = nodeInfo } - logger.Debug("assignmons: mons have been assigned to nodes") + logger.Debug("assignmons: mons have been scheduled") return nil } @@ -521,7 +617,7 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e // Ensure each of the mons have been created. If already created, it will be a no-op. for i := 0; i < len(mons); i++ { node, _ := c.mapping.Node[mons[i].DaemonName] - err := c.startMon(mons[i], node.Hostname) + err := c.startMon(mons[i], node) if err != nil { return fmt.Errorf("failed to create mon %s. %+v", mons[i].DaemonName, err) } @@ -652,13 +748,42 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error { return nil } -func (c *Cluster) startMon(m *monConfig, hostname string) error { - +// startMon creates or updates a monitor deployment. +// +// The node parameter specifies the node to be used as a node selector on the +// monitor pod. It is the result of scheduling a canary pod: see +// scheduleMonitor() for more details on scheduling. +// +// The node parameter is optional. When the parameter is nil it indicates that +// the pod should not use a node selector, and should instead rely on k8s to +// perform scheduling. +// +// The following outlines the different scenarios that exist and how deployments +// should be configured w.r.t. scheduling and the use of a node selector. +// +// 1) if HostNetworking -> always use node selector. we do not want to change +// the IP address of a monitor as it is wrapped up in the monitor's identity. +// with host networking we use node selector to ensure a stable IP for each +// monitor. see scheduleMonitor() comment for more details. +// +// Note: an important assumption is that HostNetworking setting does not +// change once a cluster is created. +// +// 2) if *not* HostNetworking -> stable IP from service; may avoid node selector +// a) when creating a new deployment +// - if HostPath -> use node selector for storage/node affinity +// - if PVC -> node selector is not required +// +// b) when updating a deployment +// - if HostPath -> leave node selector as is +// - if PVC -> remove node selector, if present +// +func (c *Cluster) startMon(m *monConfig, node *NodeInfo) error { // check if the monitor deployment already exists. if the deployment does // exist, also determine if it using pvc storage. pvcExists := false deploymentExists := false - d := c.makeDeployment(m, hostname) + d := c.makeDeployment(m) existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.Name, metav1.GetOptions{}) if err == nil { deploymentExists = true @@ -683,7 +808,22 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error { logger.Debugf("adding host path volume source to mon deployment %s", d.Name) } + // placement settings from the CRD + p := cephv1.GetMonPlacement(c.spec.Placement) + if deploymentExists { + // the existing deployment may have a node selector. if the cluster + // isn't using host networking and the deployment is using pvc storage, + // then the node selector can be removed. this may happen after + // upgrading the cluster with the k8s scheduling support for monitors. + if c.Network.IsHost() || !pvcExists { + p.PodAffinity = nil + p.PodAntiAffinity = nil + c.setPodPlacement(&d.Spec.Template.Spec, p, + existingDeployment.Spec.Template.Spec.NodeSelector) + } else { + c.setPodPlacement(&d.Spec.Template.Spec, p, nil) + } return c.updateMon(m, d) } @@ -702,6 +842,15 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error { } } + if node == nil { + c.setPodPlacement(&d.Spec.Template.Spec, p, nil) + } else { + p.PodAffinity = nil + p.PodAntiAffinity = nil + c.setPodPlacement(&d.Spec.Template.Spec, p, + map[string]string{v1.LabelHostname: node.Hostname}) + } + logger.Debugf("Starting mon: %+v", d.Name) _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d) if err != nil { diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 43eeaebfd..3c2282074 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -122,7 +122,6 @@ func newCluster(context *clusterd.Context, namespace string, network cephv1.Netw monTimeoutList: map[string]time.Time{}, mapping: &Mapping{ Node: map[string]*NodeInfo{}, - Port: map[string]int32{}, }, ownerRef: metav1.OwnerReference{}, } @@ -241,7 +240,7 @@ func TestSaveMonEndpoints(t *testing.T) { cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, "a=1.2.3.1:6789", cm.Data[EndpointDataKey]) - assert.Equal(t, `{"node":{},"port":{}}`, cm.Data[MappingKey]) + assert.Equal(t, `{"node":{}}`, cm.Data[MappingKey]) assert.Equal(t, "-1", cm.Data[MaxMonIDKey]) // update the config map @@ -252,14 +251,13 @@ func TestSaveMonEndpoints(t *testing.T) { Address: "1.1.1.1", Hostname: "myhost", } - c.mapping.Port["node0"] = int32(12345) err = c.saveMonConfig() assert.Nil(t, err) cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, "a=2.3.4.5:6789", cm.Data[EndpointDataKey]) - assert.Equal(t, `{"node":{"a":{"Name":"node0","Hostname":"myhost","Address":"1.1.1.1"}},"port":{"node0":12345}}`, cm.Data[MappingKey]) + assert.Equal(t, `{"node":{"a":{"Name":"node0","Hostname":"myhost","Address":"1.1.1.1"}}}`, cm.Data[MappingKey]) assert.Equal(t, "2", cm.Data[MaxMonIDKey]) } @@ -344,168 +342,3 @@ func TestMonFoundInQuorum(t *testing.T) { assert.True(t, monFoundInQuorum("c", response)) assert.False(t, monFoundInQuorum("d", response)) } - -// no node choice can be made when there are no nodes -func TestScheduleMonitorEmpty(t *testing.T) { - nodeZones := [][]NodeUsage{} - mon := &monConfig{DaemonName: "a"} - // no zones - assert.Nil(t, scheduleMonitor(mon, nodeZones)) - // 1 zone no mons - nodeZones = append(nodeZones, []NodeUsage{}) - assert.Nil(t, scheduleMonitor(mon, nodeZones)) - // 2 zones no mons - nodeZones = append(nodeZones, []NodeUsage{}) - assert.Nil(t, scheduleMonitor(mon, nodeZones)) -} - -// only valid nodes should be chosen -func TestScheduleMonitorInvalidNodes(t *testing.T) { - // 1 zone with 1 invalid empty node - nodeZones := [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - }, - } - mon := &monConfig{DaemonName: "a"} - assert.Nil(t, scheduleMonitor(mon, nodeZones)) - - // 1 zone with 2 invalid empty node - nodeZones = [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - }, - } - assert.Nil(t, scheduleMonitor(mon, nodeZones)) - - // 2 zone with 2 invalid empty node each - nodeZones = [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - }, - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - }, - } - assert.Nil(t, scheduleMonitor(mon, nodeZones)) -} - -func TestScheduleMonitor(t *testing.T) { - // 1 zone, 1 valid, empty node -> only one choice - nodeZones := [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - } - mon := &monConfig{DaemonName: "a"} - assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) - - // still the only choice even if not empty - nodeZones = [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, - }, - } - assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) - - // scheduler prefers the node with the least number of mons - nodeZones = [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, - }, - } - assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones)) - - // the scheduler prefers nodes with the least number of mons, not zones with - // the leads number of mons (zero mons is a special case: tested below...) - nodeZones = [][]NodeUsage{ - // 24 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 4, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, - }, - // 10 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, - }, - } - // choose the node with 4 mons - assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones)) - - // same as before, the target mon is in the second zone - nodeZones = [][]NodeUsage{ - // 6 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, - }, - // 10 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 9, MonValid: true}, - }, - } - // choose the node with 1 mon - assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) - - // prefers a zone with zero mons to spread across failure domains - nodeZones = [][]NodeUsage{ - // 0 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - // 1 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, - }, - } - // choose the zone with zero mons - assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) - - // same as before, different zone - nodeZones = [][]NodeUsage{ - // 0 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - // 1 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - } - // choose the zone with zero mons - assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) - - // invalid nodes aren't schedulable, but if they have mons, that factors - // into the decision. in this case it means the zone isn't really empty - nodeZones = [][]NodeUsage{ - // 0 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - // invalid node, but has a mon -> zone is not empty - NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: false}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - // 1 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - } - // choose the zone with zero mons - assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) -} diff --git a/pkg/operator/ceph/cluster/mon/node.go b/pkg/operator/ceph/cluster/mon/node.go index b8c2a74e3..6b33d28ef 100644 --- a/pkg/operator/ceph/cluster/mon/node.go +++ b/pkg/operator/ceph/cluster/mon/node.go @@ -124,76 +124,3 @@ func getNodeInfoFromNode(n v1.Node) (*NodeInfo, error) { } return nr, nil } - -// Returns a hierarchical representation of the nodes in the cluster organized -// by zone failure domain and annotated with the number of monitor pods assigned -// to each node and if the node is valid for scheduling monitor pods. The -// purpose of this structure is to provide a unified view of information -// required for making monitor scheduling decisions. -func (c *Cluster) getNodeMonUsage() ([][]NodeUsage, error) { - // get all k8s node objects - nodeOptions := metav1.ListOptions{} - nodeOptions.TypeMeta.Kind = "Node" - nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions) - if err != nil { - return nil, err - } - - // get all pod objects labeled as a monitor - podOptions := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)} - pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(podOptions) - if err != nil { - return nil, err - } - - // generate a list of nodes that includes the number of monitor pods - // assigned to each node. this is equivalent to: - // - // SELECT Node, Count(*) FROM Nodes - // JOIN Pods ON Nodes.Hostname == Pods.Hostname - // GROUP BY Node - // - nodeUsages := []NodeUsage{} - for i, node := range nodes.Items { - valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - continue - } - nodeUsage := NodeUsage{Node: &nodes.Items[i], MonCount: 0, MonValid: valid} - for _, pod := range pods.Items { - hostname := pod.Spec.NodeSelector[v1.LabelHostname] - if node.Name == hostname || node.Labels[v1.LabelHostname] == hostname { - nodeUsage.MonCount++ - } - } - nodeUsages = append(nodeUsages, nodeUsage) - } - - // generate a hierarchical representation of the nodes grouped by the name - // of the node's zone failure domain label. the zone label named "" is used - // for nodes without a zone annotation. - nodesByZone := make(map[string][]NodeUsage) - for _, nodeUsage := range nodeUsages { - zone, ok := nodeUsage.Node.Labels["failure-domain.beta.kubernetes.io/zone"] - if !ok { - zone = "" - } - nodesByZone[zone] = append(nodesByZone[zone], nodeUsage) - } - - // compute the final form used for scheduling. this form is an array of - // nodes by zone, with non-labeled nodes appearing in the final zone. this - // organization reflects the preference made by the scheduler of using nodes - // with zone annotations before non-labeled nodes. - res := [][]NodeUsage{} - for zone, nodeUsage := range nodesByZone { - if zone == "" { - res = append(res, nodeUsage) - } else { - res = append([][]NodeUsage{nodeUsage}, res...) - } - } - - return res, nil -} diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index 5b3d78c36..7df406102 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -32,110 +32,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestGetNodeMonUsageValidNode(t *testing.T) { - clientset := test.New(2) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - conditions := v1.NodeCondition{Type: v1.NodeOutOfDisk} - node.Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}} - clientset.CoreV1().Nodes().Update(node) - - // nodes are reported with correct valid flag - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.False(t, nodeZones[0][0].MonValid) - assert.True(t, nodeZones[0][1].MonValid) -} - -func TestGetNodeMonUsageMonCount(t *testing.T) { - clientset := test.New(2) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - // 3 mons on node0 - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - - monIDs := []string{"a", "b", "c"} - for i := 0; i < 3; i++ { - pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name) - _, err = clientset.CoreV1().Pods(c.Namespace).Create(pod) - assert.Nil(t, err) - } - - // 2 mons on node1 - node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - - monIDs = []string{"d", "e"} - for i := 0; i < 2; i++ { - pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name) - _, err = clientset.CoreV1().Pods(c.Namespace).Create(pod) - assert.Nil(t, err) - } - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 1, len(nodeZones)) - if nodeZones[0][0].Node.Name == "node0" { - assert.Equal(t, 3, nodeZones[0][0].MonCount) - assert.Equal(t, 2, nodeZones[0][1].MonCount) - } else { - assert.Equal(t, 2, nodeZones[0][1].MonCount) - assert.Equal(t, 3, nodeZones[0][0].MonCount) - } -} - -func TestTaintedNodes(t *testing.T) { - clientset := test.New(4) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - // mark a node as unschedulable - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Spec.Unschedulable = true - clientset.CoreV1().Nodes().Update(node) - - // taint - node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Spec.Taints = []v1.Taint{ - {Effect: v1.TaintEffectNoSchedule}, - } - clientset.CoreV1().Nodes().Update(node) - - // taint - node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Spec.Taints = []v1.Taint{ - {Effect: v1.TaintEffectPreferNoSchedule}, - } - clientset.CoreV1().Nodes().Update(node) - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 1, len(nodeZones)) - validCount := 0 - for zi := range nodeZones { - for ni := range nodeZones[zi] { - if nodeZones[zi][ni].MonValid { - validCount++ - } - } - } - assert.Equal(t, 1, validCount) -} - func TestNodeAffinity(t *testing.T) { clientset := test.New(4) c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) @@ -160,30 +56,17 @@ func TestNodeAffinity(t *testing.T) { } // label nodes so they appear as not scheduable / invalid - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + node, _ := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) node.Labels = map[string]string{"label": "foo"} clientset.CoreV1().Nodes().Update(node) - node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) + node, _ = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) node.Labels = map[string]string{"label": "bar"} clientset.CoreV1().Nodes().Update(node) - node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) + node, _ = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) node.Labels = map[string]string{"label": "baz"} clientset.CoreV1().Nodes().Update(node) - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 1, len(nodeZones)) - validCount := 0 - for zi := range nodeZones { - for ni := range nodeZones[zi] { - if nodeZones[zi][ni].MonValid { - validCount++ - } - } - } - assert.Equal(t, 2, validCount) } // this tests can 3 mons with hostnetworking on the same host is rejected @@ -282,22 +165,8 @@ func TestHostNetwork(t *testing.T) { c.Network.HostNetwork = true - nodes := []v1.Node{} - nodeZones, err := c.getNodeMonUsage() - assert.Nil(t, err) - - for zi := range nodeZones { - for _, nodeUsage := range nodeZones[zi] { - if nodeUsage.MonCount == 0 || c.spec.Mon.AllowMultiplePerNode { - nodes = append(nodes, *nodeUsage.Node) - } - } - } - - assert.Equal(t, 3, len(nodes)) - monConfig := testGenMonConfig("c") - pod := c.makeMonPod(monConfig, nodes[2].Name) + pod := c.makeMonPod(monConfig) assert.NotNil(t, pod) assert.Equal(t, true, pod.Spec.HostNetwork) assert.Equal(t, v1.DNSClusterFirstWithHostNet, pod.Spec.DNSPolicy) @@ -308,7 +177,7 @@ func TestHostNetwork(t *testing.T) { assert.Equal(t, "arg not found: --public-bind-addr", message) monConfig.Port = 6790 - pod = c.makeMonPod(monConfig, nodes[2].Name) + pod = c.makeMonPod(monConfig) val, message = extractArgValue(pod.Spec.Containers[0].Args, "--public-addr") assert.Equal(t, "2.4.6.3:6790", val, message) assert.NotNil(t, pod) @@ -399,134 +268,3 @@ func TestTargetMonCount(t *testing.T) { assert.Equal(t, 7, target) logger.Infof(msg) } - -// mon node usage should return no zones if there are no nodes -func TestGetNodeMonUsageNoNodes(t *testing.T) { - clientset := test.New(0) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, len(nodeZones), 0) -} - -// all nodes without zone annotations are in the same zone -func TestGetNodeMonUsageNoZoneLabels(t *testing.T) { - for i := 1; i < 5; i++ { - clientset := test.New(i) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, len(nodeZones), 1) - assert.Equal(t, len(nodeZones[0]), i) - } -} - -// nodes are partitioned into separate zones -func TestGetNodeMonUsageZoneSpread(t *testing.T) { - clientset := test.New(5) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - // 1 node labeled -> 1 zone + the rest in the unlabeled zone - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"} - clientset.CoreV1().Nodes().Update(node) - - // the zones are unlabeled and sort order isn't guaranteed. so instead we - // assert on expected count of zones and collection of zone sizes. - zoneCounts := map[int]int{} - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 2, len(nodeZones)) - for _, zones := range nodeZones { - zoneSize := len(zones) - if _, ok := zoneCounts[zoneSize]; ok { - zoneCounts[zoneSize]++ - } else { - zoneCounts[zoneSize] = 1 - } - } - assert.Equal(t, map[int]int{1: 1, 4: 1}, zoneCounts) - - // 2 nodes with same label, 3 unlabeled - node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"} - clientset.CoreV1().Nodes().Update(node) - - // reset - zoneCounts = map[int]int{} - - nodeZones, err = c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 2, len(nodeZones)) - for _, zones := range nodeZones { - zoneSize := len(zones) - if _, ok := zoneCounts[zoneSize]; ok { - zoneCounts[zoneSize]++ - } else { - zoneCounts[zoneSize] = 1 - } - } - assert.Equal(t, map[int]int{2: 1, 3: 1}, zoneCounts) - - // 2 nodes with same label, 2 with distinct labels, 1 unlabeled - node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"} - clientset.CoreV1().Nodes().Update(node) - - node, err = clientset.CoreV1().Nodes().Get("node3", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z2"} - clientset.CoreV1().Nodes().Update(node) - - // reset - zoneCounts = map[int]int{} - - nodeZones, err = c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 4, len(nodeZones)) - for _, zones := range nodeZones { - zoneSize := len(zones) - if _, ok := zoneCounts[zoneSize]; ok { - zoneCounts[zoneSize]++ - } else { - zoneCounts[zoneSize] = 1 - } - } - assert.Equal(t, map[int]int{2: 1, 1: 3}, zoneCounts) - - // no unlabeled zones - node, err = clientset.CoreV1().Nodes().Get("node4", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"} - clientset.CoreV1().Nodes().Update(node) - - // reset - zoneCounts = map[int]int{} - - nodeZones, err = c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 3, len(nodeZones)) - for _, zones := range nodeZones { - zoneSize := len(zones) - if _, ok := zoneCounts[zoneSize]; ok { - zoneCounts[zoneSize]++ - } else { - zoneCounts[zoneSize] = 1 - } - } - assert.Equal(t, map[int]int{2: 2, 1: 1}, zoneCounts) -} diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index 954f1c6d3..5c9cb50d7 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -48,9 +48,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { }, } k8sutil.SetOwnerRef(&svcDef.ObjectMeta, &c.ownerRef) - if c.Network.IsHost() { - svcDef.Spec.ClusterIP = v1.ClusterIPNone - } // If deploying Nautilus or newer we need a new port for the monitor service if c.ClusterInfo.CephVersion.IsAtLeastNautilus() { diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index f920a25d7..bec0d11d3 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -22,6 +22,7 @@ import ( "path" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rook "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" @@ -52,7 +53,7 @@ func (c *Cluster) getLabels(daemonName string) map[string]string { return labels } -func (c *Cluster) makeDeployment(monConfig *monConfig, hostname string) *apps.Deployment { +func (c *Cluster) makeDeployment(monConfig *monConfig) *apps.Deployment { d := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: monConfig.ResourceName, @@ -65,7 +66,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, hostname string) *apps.De opspec.AddCephVersionLabelToDeployment(c.ClusterInfo.CephVersion, d) k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef) - pod := c.makeMonPod(monConfig, hostname) + pod := c.makeMonPod(monConfig) replicaCount := int32(1) d.Spec = apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -134,7 +135,49 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig) (*v1.PersistentVolumeClaim, er * Pod spec */ -func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { +func (c *Cluster) setPodPlacement(pod *v1.PodSpec, p rook.Placement, nodeSelector map[string]string) { + p.ApplyToPodSpec(pod) + pod.NodeSelector = nodeSelector + + // when a node selector is being used, skip the affinity business below + if nodeSelector != nil { + return + } + + // label selector for monitors used in anti-affinity rules + monAntiAffinity := v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + k8sutil.AppAttr: AppName, + }, + }, + TopologyKey: v1.LabelHostname, + } + + // set monitor pod anti-affinity rules. when monitors should never be + // co-located (e.g. not AllowMultiplePerHost or HostNetworking) then the + // anti-affinity rule is made to be required during scheduling, otherwise it + // is merely a preferred policy. + // + // ApplyToPodSpec ensures that pod.Affinity is non-nil + if pod.Affinity.PodAntiAffinity == nil { + pod.Affinity.PodAntiAffinity = &v1.PodAntiAffinity{} + } + paa := pod.Affinity.PodAntiAffinity + + if c.Network.IsHost() || !c.spec.Mon.AllowMultiplePerNode { + paa.RequiredDuringSchedulingIgnoredDuringExecution = + append(paa.RequiredDuringSchedulingIgnoredDuringExecution, monAntiAffinity) + } else { + paa.PreferredDuringSchedulingIgnoredDuringExecution = + append(paa.PreferredDuringSchedulingIgnoredDuringExecution, v1.WeightedPodAffinityTerm{ + Weight: 50, + PodAffinityTerm: monAntiAffinity, + }) + } +} + +func (c *Cluster) makeMonPod(monConfig *monConfig) *v1.Pod { logger.Debugf("monConfig: %+v", monConfig) podSpec := v1.PodSpec{ InitContainers: []v1.Container{ @@ -145,7 +188,6 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { c.makeMonDaemonContainer(monConfig), }, RestartPolicy: v1.RestartPolicyAlways, - NodeSelector: map[string]string{v1.LabelHostname: hostname}, Volumes: opspec.DaemonVolumesBase(monConfig.DataPathMap, keyringStoreName), HostNetwork: c.Network.IsHost(), } @@ -153,13 +195,6 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet } - // apply the pod placement if specified in the crd - // remove Pod (anti-)affinity because we have our own placement logic - p := cephv1.GetMonPlacement(c.spec.Placement) - p.PodAffinity = nil - p.PodAntiAffinity = nil - p.ApplyToPodSpec(&podSpec) - pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: monConfig.ResourceName, diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index e876b1f43..bf4ebb3c4 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -21,6 +21,7 @@ import ( "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rook "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" @@ -64,7 +65,7 @@ func testPodSpec(t *testing.T, monID string, pvc bool) { } monConfig := testGenMonConfig(monID) - d := c.makeDeployment(monConfig, "node0") + d := c.makeDeployment(monConfig) assert.NotNil(t, d) if pvc { @@ -145,3 +146,74 @@ func TestDeploymentPVCSpec(t *testing.T) { assert.NoError(t, err) assert.Equal(t, pvc.Spec.Resources.Requests[v1.ResourceStorage], req) } + +func testPodSpecPlacement(t *testing.T, hostNet, allowMulti bool, req, pref int, placement *rook.Placement) { + clientset := testop.New(1) + c := New( + &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"}, + "ns", + "/var/lib/rook", + cephv1.NetworkSpec{HostNetwork: hostNet}, + metav1.OwnerReference{}, + &sync.Mutex{}, + ) + + setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: allowMulti}, "rook/rook:myversion") + monConfig := testGenMonConfig("a") + + if placement != nil { + c.spec.Placement = rook.PlacementSpec{} + c.spec.Placement["mon"] = *placement + } + + d := c.makeDeployment(monConfig) + assert.NotNil(t, d) + + p := cephv1.GetMonPlacement(c.spec.Placement) + c.setPodPlacement(&d.Spec.Template.Spec, p, nil) + + // should have a required anti-affnity and no preferred anti-affinity + assert.Equal(t, + req, + len(d.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution)) + assert.Equal(t, + pref, + len(d.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution)) +} + +func makePlacement() rook.Placement { + return rook.Placement{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + TopologyKey: v1.LabelZoneFailureDomain, + }, + }, + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + PodAffinityTerm: v1.PodAffinityTerm{ + TopologyKey: v1.LabelZoneFailureDomain, + }, + }, + }, + }, + } +} + +func TestPodSpecPlacement(t *testing.T) { + // no placement settings in the crd + testPodSpecPlacement(t, true, true, 1, 0, nil) + testPodSpecPlacement(t, true, false, 1, 0, nil) + testPodSpecPlacement(t, false, true, 0, 1, nil) + testPodSpecPlacement(t, false, false, 1, 0, nil) + + // crd has other preferred and required anti-affinity setting + p := makePlacement() + testPodSpecPlacement(t, true, true, 2, 1, &p) + p = makePlacement() + testPodSpecPlacement(t, true, false, 2, 1, &p) + p = makePlacement() + testPodSpecPlacement(t, false, true, 1, 2, &p) + p = makePlacement() + testPodSpecPlacement(t, false, false, 2, 1, &p) +} From e2fccdf03b4887a90892ef6c493a3f25cbbd23dd Mon Sep 17 00:00:00 2001 From: Noah Watkins Date: Fri, 23 Aug 2019 13:59:33 -0700 Subject: [PATCH 3/3] ceph: remove the CRD option Mon.PreferredCount Signed-off-by: Noah Watkins --- Documentation/ceph-cluster-crd.md | 4 -- PendingReleaseNotes.md | 1 + .../charts/rook-ceph/templates/resources.yaml | 4 -- cluster/examples/kubernetes/ceph/common.yaml | 4 -- design/ceph-mon-pv.md | 6 +- pkg/apis/ceph.rook.io/v1/types.go | 1 - pkg/operator/ceph/cluster/mon/health.go | 7 +-- pkg/operator/ceph/cluster/mon/mon.go | 8 +-- pkg/operator/ceph/cluster/mon/node.go | 62 +------------------ pkg/operator/ceph/cluster/mon/node_test.go | 52 ---------------- tests/framework/installer/ceph_manifests.go | 4 -- 11 files changed, 8 insertions(+), 145 deletions(-) diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 6559f6ccc..08c6a44d4 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -130,10 +130,6 @@ For more details on the mons and when to choose a number other than `3`, see the ### Mon Settings - `count`: Set the number of mons to be started. The number should be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`. -- `preferredCount`: If you want to increase the number of mons when the number of nodes increases, set the `preferredCount` to be larger than the `count`. For example, if your cluster -starts with three nodes, but might grow to more than five nodes, you might want five mons running after the other nodes come online. In this case, set `count: 3` and `preferredCount: 5`. -When the operator sees the new nodes come online, the number of mons will increase to the preferred count. If the number of nodes decreases below the `preferredCount`, the operator will -reduce the number of mons back to `count`. If `allowMultiplePerNode: true` (for testing scenarios), the number of mons will always use `preferredCount` if set. - `allowMultiplePerNode`: Enable (`true`) or disable (`false`) the placement of multiple mons on one node. Default is `false`. - `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 444ecd860..2300bc174 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -52,6 +52,7 @@ an example usage - The minimum version supported by Rook is Ceph Mimic v13.2.4. Before upgrading to v1.1 it is required to update the version of Ceph to at least this version. - The CSI driver is enabled by default. Documentation has been changed significantly for block and filesystem to use the CSI driver instead of flex. While the flex driver is still supported, it is anticipated to be deprecated soon. +- The `Mon.PreferredCount` setting has been removed. ## Known Issues diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 71027a620..a8c0fa7c7 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -47,10 +47,6 @@ spec: maximum: 9 minimum: 1 type: integer - preferredCount: - maximum: 9 - minimum: 0 - type: integer required: - count network: diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 1c9c66895..59cb07cc6 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -66,10 +66,6 @@ spec: maximum: 9 minimum: 0 type: integer - preferredCount: - maximum: 9 - minimum: 0 - type: integer network: properties: hostNetwork: diff --git a/design/ceph-mon-pv.md b/design/ceph-mon-pv.md index 9a513deaf..8599fe415 100644 --- a/design/ceph-mon-pv.md +++ b/design/ceph-mon-pv.md @@ -23,7 +23,6 @@ claim template that is used to generate PVCs for monitor database storage. ```go type MonSpec struct { Count int `json:"count"` - PreferredCount int `json:"preferredCount"` AllowMultiplePerNode bool `json:"allowMultiplePerNode"` VolumeClaimTemplate *v1.PersistentVolumeClaim } @@ -128,6 +127,8 @@ has eviction policies in the future we could then make use of it (e.g. with ### Target Monitor Count +The `PreferredCount` feature has been removed. + The CRD monitor count specifies a target minimum number of monitors to maintain. Additionally, a preferred count is available which will be the desired number of sufficient number of nodes are available. Unfortunately, this calculation @@ -135,6 +136,3 @@ relies on determining if monitor pods may be placed on a node, requiring knowledge of the scheduling policy and algorithm. The scenario to watch out for is an endless loop in which the health check is determining a bad placement but the k8s schedule thinks otherwise. - -**TODO**: one approach here may be to schedule the desired number of canary pods -to check for capacity before proceeding through monitor orchestration. diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 6739cca79..f2cbfa000 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -150,7 +150,6 @@ const ( type MonSpec struct { Count int `json:"count,omitempty"` - PreferredCount int `json:"preferredCount,omitempty"` AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"` VolumeClaimTemplate *v1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"` } diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index d08f2a55f..e47eab881 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -92,11 +92,8 @@ func (c *Cluster) checkHealth() error { // Use a local mon count in case the user updates the crd in another goroutine. // We need to complete a health check with a consistent value. - desiredMonCount, msg, err := c.getTargetMonCount() - if err != nil { - return fmt.Errorf("failed to get target mon count. %+v", err) - } - logger.Debugf(msg) + desiredMonCount := c.spec.Mon.Count + logger.Debugf("targeting the mon count %d", desiredMonCount) // Source of truth of which mons should exist is our *clusterInfo* monsNotFound := map[string]interface{}{} diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index e0099ac1b..aa53dc755 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -202,14 +202,10 @@ func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string, return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err) } - targetCount, msg, err := c.getTargetMonCount() - if err != nil { - return nil, fmt.Errorf("failed to get target mon count. %+v", err) - } - logger.Infof(msg) + logger.Infof("targeting the mon count %d", c.spec.Mon.Count) // create the mons for a new cluster or ensure mons are running in an existing cluster - return c.ClusterInfo, c.startMons(targetCount) + return c.ClusterInfo, c.startMons(c.spec.Mon.Count) } func (c *Cluster) startMons(targetCount int) error { diff --git a/pkg/operator/ceph/cluster/mon/node.go b/pkg/operator/ceph/cluster/mon/node.go index 6b33d28ef..2bee58df1 100644 --- a/pkg/operator/ceph/cluster/mon/node.go +++ b/pkg/operator/ceph/cluster/mon/node.go @@ -19,10 +19,7 @@ package mon import ( "fmt" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/operator/k8sutil" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/api/core/v1" ) // NodeUsage is a mapping between a Node and computed metadata about the node @@ -36,63 +33,6 @@ type NodeUsage struct { MonValid bool } -// Get the number of mons that the operator should be starting -func (c *Cluster) getTargetMonCount() (int, string, error) { - - // Get the full list of k8s nodes - nodeOptions := metav1.ListOptions{} - nodeOptions.TypeMeta.Kind = "Node" - nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions) - if err != nil { - return 0, "", fmt.Errorf("failed to get nodes. %+v", err) - } - - // Get the nodes where it is possible to place mons - availableNodes := []v1.Node{} - for _, node := range nodes.Items { - valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - } else if valid { - availableNodes = append(availableNodes, node) - } - } - - target, msg := calcTargetMonCount(len(availableNodes), c.spec.Mon) - return target, msg, nil -} - -func calcTargetMonCount(nodes int, spec cephv1.MonSpec) (int, string) { - minTarget := spec.Count - preferredTarget := spec.PreferredCount - - if preferredTarget <= minTarget { - msg := fmt.Sprintf("targeting the mon count %d", minTarget) - return minTarget, msg - } - if nodes >= preferredTarget { - msg := fmt.Sprintf("targeting the preferred mon count %d since there are %d available nodes", preferredTarget, nodes) - return preferredTarget, msg - } - if spec.AllowMultiplePerNode { - msg := fmt.Sprintf("targeting the preferred mon count %d even if not that many nodes since multiple mons are allowed per node", preferredTarget) - return preferredTarget, msg - } - if nodes <= minTarget { - msg := fmt.Sprintf("targeting the min mon count %d since there are only %d available nodes", minTarget, nodes) - return minTarget, msg - } - - // There are between minTarget and preferredTarget nodes. Find an odd number for mons closest to the number of nodes. - intermediate := nodes - if intermediate%2 == 0 { - // Decrease to an odd number if there are an even number of nodes - intermediate-- - } - msg := fmt.Sprintf("targeting the calculated mon count %d since there are %d available nodes", intermediate, nodes) - return intermediate, msg -} - // Look up the immutable node name from the hostname label func getNodeNameFromHostname(nodes *v1.NodeList, hostname string) (string, bool) { for _, node := range nodes.Items { diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index 7df406102..c671847db 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -216,55 +216,3 @@ func TestGetNodeInfoFromNode(t *testing.T) { assert.Equal(t, "1.1.1.1", info.Address) } - -func TestTargetMonCount(t *testing.T) { - // only a single node and min 1 - spec := cephv1.MonSpec{Count: 1, PreferredCount: 0, AllowMultiplePerNode: false} - nodes := 1 - target, msg := calcTargetMonCount(nodes, spec) - assert.Equal(t, 1, target) - logger.Infof(msg) - - // preferred 3 mons, but only one node available - spec.PreferredCount = 3 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 1, target) - logger.Infof(msg) - - // preferred 3 mons, and 3 nodes available - nodes = 3 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 3, target) - logger.Infof(msg) - - // get an intermediate odd number since we have several options between the min and the node count - nodes = 4 - spec.Count = 1 - spec.PreferredCount = 5 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 3, target) - logger.Infof(msg) - nodes = 5 - spec.PreferredCount = 6 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 5, target) - logger.Infof(msg) - nodes = 6 - spec.PreferredCount = 7 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 5, target) - logger.Infof(msg) - nodes = 7 - spec.PreferredCount = 7 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 7, target) - logger.Infof(msg) - - // multiple allowed per node, so we always returned preferred - nodes = 1 - spec.PreferredCount = 7 - spec.AllowMultiplePerNode = true - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 7, target) - logger.Infof(msg) -} diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 8c9448b6d..e0ae986ba 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -120,10 +120,6 @@ spec: maximum: 9 minimum: 1 type: integer - preferredCount: - maximum: 9 - minimum: 0 - type: integer required: - count network: