diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 06085dc6c..221b71a3d 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -79,6 +79,10 @@ spec: # operator: Exists # The above placement information can also be specified for mon, osd, and mgr components # mon: +# Monitor deployments may contain an anti-affinity rule for avoiding monitor +# colocation on the same node. This is a required rule when host network is used +# or when AllowMultiplePerNode is false. Otherwise this anti-affinity rule is a +# preferred rule with weight: 50. # osd: # mgr: annotations: diff --git a/design/ceph-mon-pv.md b/design/ceph-mon-pv.md index 4d04dbccb..9a513deaf 100644 --- a/design/ceph-mon-pv.md +++ b/design/ceph-mon-pv.md @@ -89,9 +89,52 @@ scheduling implementation in Rook. These restrictions will be removed or significantly relaxed once monitor scheduling is moved under control of Kubernetes itself (ongoing work). -## Migration +## Scheduling -Monitor migration is currently not implemented. When nodes are decommissioned, -for example, any monitors on the nodes are migrated using a fail over mechanism -rather than also migrating the PVC. This support is currently being worked on in -conjunction with the enhancements to scheduling. +In previous versions of Rook the operator made explicit scheduling (placement) +decisions when creating monitor deployments. These decisions were made by +implementing a custom scheduling algorithm, and using the pod node selector to +enforce the placement decision. Unfortunately, schedulers are difficult to +write correctly, and manage. Furthermore, by maintaining a separate scheduler +from Kubernetes global policies are difficult to achieve. + +Despite the benefits of using the Kubernetes scheduler, there are important use +cases for using a node selector on a monitor deployment: pinning a monitor to a +node when HostPath-based storage is used. In this case Rook must prevent k8s +from moving a pod away from the node that contains its storage. The node +selector is used to enforce this affinity. Unfortunately, node selector use is +mutually exclusive with kubernetes scheduling---a pod cannot be scheduled by +Kubernetes and then atomically have its affinity set to that placement decision. + +The workaround in Rook is to use a temporary _canary pod_ that is scheduled by +Kubernetes, but whose placement is enforced by Rook. The canary deployment is a +deployment configured identically to a monitor deployment, except the container +entrypoints have no side affects. The canary deployments are used to solve a +fundamental bootstrapping issue: we want to avoid making explicit scheduling +decisions in Rook, but in some configurations a node selector needs to be used +to pin a monitor to a node. + +### Health checks + +Previous versions of Rook performed periodic health checks that included checks +on monitor health as well as looking for scheduling violations. The health +checks related to scheduling violations have been removed. Fundamentally a +placement violation requires understanding or accessing the scheduling algorithm. + +The rescheduling or eviction aspects of Rook's scheduling caused more problems +than it helped, so going with K8s scheduling is the right thing. If/when K8s +has eviction policies in the future we could then make use of it (e.g. with +`*RequiredDuringExecution` variants of anti-affinity rules are available). + +### Target Monitor Count + +The CRD monitor count specifies a target minimum number of monitors to maintain. +Additionally, a preferred count is available which will be the desired number of +sufficient number of nodes are available. Unfortunately, this calculation +relies on determining if monitor pods may be placed on a node, requiring +knowledge of the scheduling policy and algorithm. The scenario to watch out for +is an endless loop in which the health check is determining a bad placement but +the k8s schedule thinks otherwise. + +**TODO**: one approach here may be to schedule the desired number of canary pods +to check for capacity before proceeding through monitor orchestration. diff --git a/pkg/operator/ceph/cluster/mon/config.go b/pkg/operator/ceph/cluster/mon/config.go index f62a411b9..d02a8523c 100644 --- a/pkg/operator/ceph/cluster/mon/config.go +++ b/pkg/operator/ceph/cluster/mon/config.go @@ -81,7 +81,6 @@ func CreateOrLoadClusterInfo(context *clusterd.Context, namespace string, ownerR maxMonID := -1 monMapping := &Mapping{ Node: map[string]*NodeInfo{}, - Port: map[string]int32{}, } secrets, err := context.Clientset.CoreV1().Secrets(namespace).Get(AppName, metav1.GetOptions{}) @@ -137,7 +136,6 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string maxMonID := -1 monMapping := &Mapping{ Node: map[string]*NodeInfo{}, - Port: map[string]int32{}, } cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 0de6f66b9..d08f2a55f 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -25,11 +25,8 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" cephutil "github.com/rook/rook/pkg/daemon/ceph/util" - "github.com/rook/rook/pkg/operator/k8sutil" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - scheduler "k8s.io/kubernetes/pkg/scheduler/api" ) var ( @@ -172,13 +169,6 @@ func (c *Cluster) checkHealth() error { return nil } - // find any mons that invalidate our placement policy, and if necessary, - // reschedule them to other nodes. - done, err := c.resolveInvalidMonitorPlacement(desiredMonCount) - if done || err != nil { - return err - } - // create/start new mons when there are fewer mons than the desired count in the CRD if len(status.MonMap.Mons) < desiredMonCount { logger.Infof("adding mons. currently %d mons are in quorum and the desired count is %d.", len(status.MonMap.Mons), desiredMonCount) @@ -220,16 +210,10 @@ func (c *Cluster) failoverMon(name string) error { m := c.newMonConfig(c.maxMonID + 1) logger.Infof("starting new mon: %+v", m) - // Create the service endpoint - serviceIP, err := c.createService(m) - if err != nil { - return fmt.Errorf("failed to create mon service. %+v", err) - } - mConf := []*monConfig{m} // Assign the pod to a node - if err = c.assignMons(mConf); err != nil { + if err := c.assignMons(mConf); err != nil { return fmt.Errorf("failed to place new mon on a node. %+v", err) } @@ -240,12 +224,17 @@ func (c *Cluster) failoverMon(name string) error { } m.PublicIP = node.Address } else { + // Create the service endpoint + serviceIP, err := c.createService(m) + if err != nil { + return fmt.Errorf("failed to create mon service. %+v", err) + } m.PublicIP = serviceIP } c.ClusterInfo.Monitors[m.DaemonName] = cephconfig.NewMonInfo(m.DaemonName, m.PublicIP, m.Port) // Start the deployment - if err = c.startDeployments(mConf, true); err != nil { + if err := c.startDeployments(mConf, true); err != nil { return fmt.Errorf("failed to start new mon %s. %+v", m.DaemonName, err) } @@ -279,19 +268,7 @@ func (c *Cluster) removeMon(daemonName string) error { delete(c.ClusterInfo.Monitors, daemonName) // check if a mapping exists for the mon if _, ok := c.mapping.Node[daemonName]; ok { - nodeName := c.mapping.Node[daemonName].Name delete(c.mapping.Node, daemonName) - // if node->port "mapping" has been created, decrease or delete it - if port, ok := c.mapping.Port[nodeName]; ok { - if port == DefaultMsgr1Port { - delete(c.mapping.Port, nodeName) - } - // don't clean up if a node port is higher than the default port, other - // mons could be on the same node with > DefaultPort ports, decreasing could - // cause port collisions - // This can be solved by using a map[nodeName][]int32 for the ports to - // even better check which ports are open for the HostNetwork mode - } } // Remove the service endpoint @@ -321,151 +298,6 @@ func removeMonitorFromQuorum(context *clusterd.Context, clusterName, name string return nil } -func (c *Cluster) resolveInvalidMonitorPlacement(desiredMonCount int) (bool, error) { - nodeChoice, err := c.findInvalidMonitorPlacement(desiredMonCount) - if err != nil { - return true, fmt.Errorf("failed to find invalid mon placement %+v", err) - } - - // no violation found - if nodeChoice == nil { - return false, nil - } - - for name, nodeInfo := range c.mapping.Node { - if nodeInfo.Name == nodeChoice.Node.Name { - logger.Infof("rebalance: rescheduling mon %s from node %s", name, nodeInfo.Name) - c.failMon(len(c.ClusterInfo.Monitors), desiredMonCount, name) - return true, nil - } - } - - logger.Warningf("rebalance: no mon pod found on node %s", nodeChoice.Node.Name) - - return false, nil -} - -func (c *Cluster) findInvalidMonitorPlacement(desiredMonCount int) (*NodeUsage, error) { - nodeZones, err := c.getNodeMonUsage() - if err != nil { - return nil, fmt.Errorf("failed to get node monitor usage. %+v", err) - } - - // compute two helpful global flags: - // - does an empty zone exist - // - does an empty node exist - emptyZone := false - emptyNode := false - for zi := range nodeZones { - monFoundInZone := false - for ni := range nodeZones[zi] { - nodeUsage := &nodeZones[zi][ni] - if nodeUsage.MonCount > 0 { - monFoundInZone = true - } else if nodeUsage.MonValid { - // only consider valid nodes since this flag determines if a mon - // may be scheduled on to a new node. - emptyNode = true - } - } - if !monFoundInZone { - // emptyZone is used below to imply that an empty zone exists AND - // that zone contains a valid node (i.e. a node that new pods can be - // assigned). this check enforces that assumption by checking that - // the zone contains at least one valid node. - validNodeInZone := false - for _, nodeUsage := range nodeZones[zi] { - if nodeUsage.MonValid { - validNodeInZone = true - break - } - } - if validNodeInZone { - emptyZone = true - } - logger.Debugf("rebalance: empty zone found. validNodeInZone: %t, emptyZone %t", - validNodeInZone, emptyZone) - } - } - - logger.Debugf("rebalance: desired mon count: %d, empty zone found: %t, empty node found: %t", - desiredMonCount, emptyZone, emptyNode) - - for zi := range nodeZones { - zoneMonCount := 0 - var nodeChoice *NodeUsage - - // for each node consider two cases: too many monitors are running on a - // node, or monitors are running on invalid nodes. - for ni := range nodeZones[zi] { - nodeUsage := &nodeZones[zi][ni] - zoneMonCount += nodeUsage.MonCount - - // if this node has too many monitors, and an underused node exists, - // then consider moving the monitor. note that this check is - // independent of the setting `AllowMultiplePerNode` since we also - // want to avoid in general keeping multiple monitors on one node. - if nodeUsage.MonCount > 1 && emptyNode { - logger.Infof("rebalance: chose overloaded node %s with %d mons", - nodeUsage.Node.Name, nodeUsage.MonCount) - nodeChoice = nodeUsage - break - } - - // check for mons on invalid nodes. but reschedule pod only when it - // is invalid for reasons other than schedulability which should not - // affect pods that are already scheduled. - if nodeUsage.MonCount > 0 && !nodeUsage.MonValid { - placement := cephv1.GetMonPlacement(c.spec.Placement) - placement.Tolerations = append(placement.Tolerations, - v1.Toleration{ - Key: scheduler.TaintNodeUnschedulable, - Effect: "", - Operator: v1.TolerationOpExists, - }) - valid, err := k8sutil.ValidNodeNoSched(*nodeUsage.Node, placement) - if err != nil { - logger.Warningf("rebalance: failed to validate node %s %+v", - nodeUsage.Node.Name, err) - } else if !valid { - logger.Infof("rebalance: chose invalid node %s", - nodeUsage.Node.Name) - nodeChoice = nodeUsage - break - } - } - } - - // if we didn't already select a node with a monitor for rescheduling, - // then consider that a zone may be overloaded and can be rebalanced. - // this case occurs if the zone has more than 1 monitor, and some other - // zone exists without any monitors. - if nodeChoice == nil && zoneMonCount > 1 && emptyZone { - for ni := range nodeZones[zi] { - nodeUsage := &nodeZones[zi][ni] - if nodeUsage.MonCount > 0 { - // select the node with the most monitors assigned - if nodeChoice == nil || nodeUsage.MonCount > nodeChoice.MonCount { - nodeChoice = nodeUsage - } - } - } - if nodeChoice != nil { - logger.Infof("rebalance: chose node %s from overloaded zone", - nodeChoice.Node.Name) - } - } - - if nodeChoice != nil { - return nodeChoice, nil - } - } - - logger.Debugf("rebalance: no mon placement violations or fixes available") - - return nil, nil -} - func (c *Cluster) handleExternalMonStatus(status client.MonStatusResponse) error { changed, err := c.addOrRemoveExternalMonitor(status) diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index c4264f9e7..3c059d263 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -17,7 +17,6 @@ limitations under the License. package mon import ( - "encoding/json" "fmt" "io/ioutil" "os" @@ -34,7 +33,6 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -66,9 +64,14 @@ func TestCheckHealth(t *testing.T) { Name: "node0", Address: "", } - c.mapping.Port["node0"] = DefaultMsgr1Port c.maxMonID = 4 + // mock out the scheduler to return node0 + scheduleMonitor = func(c *Cluster, mon *monConfig) (SchedulingResult, error) { + node, _ := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + return SchedulingResult{Node: node}, nil + } + err := c.checkHealth() assert.Nil(t, err) logger.Infof("mons after checkHealth: %v", c.ClusterInfo.Monitors) @@ -117,7 +120,6 @@ func TestCheckHealthNotFound(t *testing.T) { c.mapping.Node["b"] = &NodeInfo{ Name: "node0", } - c.mapping.Port["node0"] = DefaultMsgr1Port c.maxMonID = 4 c.saveMonConfig() @@ -145,304 +147,6 @@ func TestCheckHealthNotFound(t *testing.T) { } } -func TestOverloadedZoneRebalances(t *testing.T) { - clientset := test.New(3) - - // build a cluster with 3 nodes. two of the nodes are in the same AZ the - // third node is in a separate AZ. - for i := 0; i < 3; i++ { - nodeName := fmt.Sprintf("node%d", i) - node, err := clientset.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - - // zone0: node0, node1 - // zone1: node2 - var zoneName string - if i < 2 { - zoneName = "zone0" - } else { - zoneName = "zone1" - } - - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": zoneName} - clientset.CoreV1().Nodes().Update(node) - } - - executor := &exectest.MockExecutor{} - - configDir, _ := ioutil.TempDir("", "") - defer os.RemoveAll(configDir) - context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - } - - cluster := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(cluster, 2, cephv1.MonSpec{Count: 2, AllowMultiplePerNode: false}, "myversion") - cluster.waitForStart = false - defer os.RemoveAll(cluster.context.ConfigDir) - - // add mon mappings - cluster.mapping.Node["a"] = &NodeInfo{ - Name: "node0", - Hostname: "mynode0", - Address: "0.0.0.0", - } - cluster.mapping.Node["b"] = &NodeInfo{ - Name: "node1", - Hostname: "mynode1", - Address: "0.0.0.0", - } - cluster.maxMonID = 1 - cluster.saveMonConfig() - - monNames := []string{"a", "b"} - nodeNames := []string{"node0", "node1"} - for i := 0; i < len(monNames); i++ { - monConfig := testGenMonConfig(monNames[i]) - nodeName := nodeNames[i] - - d := cluster.makeDeployment(monConfig, nodeName) - _, err := clientset.AppsV1().Deployments(cluster.Namespace).Create(d) - assert.Nil(t, err) - - po := cluster.makeMonPod(monConfig, nodeName) - _, err = clientset.CoreV1().Pods(cluster.Namespace).Create(po) - assert.Nil(t, err) - } - - assert.NotContains(t, cluster.mapping.Node, "c") - assert.Equal(t, "node0", cluster.mapping.Node["a"].Name) - assert.Equal(t, "node1", cluster.mapping.Node["b"].Name) - assert.Equal(t, "mynode0", cluster.mapping.Node["a"].Hostname) - assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname) - - _, err := cluster.resolveInvalidMonitorPlacement(2) - assert.Nil(t, err) - - // mon.a was chosen from the overloaded zone 0, and a new monitor to take - // its place, mon.c was created on a node in zone 1 that had no monitors. - assert.NotContains(t, cluster.mapping.Node, "a") - assert.Equal(t, "node1", cluster.mapping.Node["b"].Name) - assert.Equal(t, "node2", cluster.mapping.Node["c"].Name) - assert.Equal(t, "mynode1", cluster.mapping.Node["b"].Hostname) -} - -func TestCheckHealthTwoMonsOneNode(t *testing.T) { - executorNextMons := false - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { - if executorNextMons { - resp := client.MonStatusResponse{Quorum: []int{0}} - resp.MonMap.Mons = []client.MonMapEntry{ - { - Name: "a", - Rank: 0, - Address: "1.2.3.4", - }, - { - Name: "c", - Rank: 0, - Address: "1.2.3.4", - }, - } - serialized, _ := json.Marshal(resp) - return string(serialized), nil - } - return clienttest.MonInQuorumResponseMany(2), nil - }, - } - clientset := test.New(1) - configDir, _ := ioutil.TempDir("", "") - defer os.RemoveAll(configDir) - context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - } - - c := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 2, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - c.waitForStart = false - defer os.RemoveAll(c.context.ConfigDir) - - // add two mons to the mapping on node0 - c.mapping.Node["a"] = &NodeInfo{ - Name: "node0", - Hostname: "mynode0", - Address: "0.0.0.0", - } - c.mapping.Node["b"] = &NodeInfo{ - Name: "node0", - Hostname: "mynode0", - Address: "0.0.0.0", - } - c.maxMonID = 1 - c.saveMonConfig() - - monNames := []string{"a", "b"} - for i := 0; i < len(monNames); i++ { - monConfig := testGenMonConfig(monNames[i]) - d := c.makeDeployment(monConfig, "node0") - _, err := clientset.AppsV1().Deployments(c.Namespace).Create(d) - assert.Nil(t, err) - po := c.makeMonPod(monConfig, "node0") - _, err = clientset.CoreV1().Pods(c.Namespace).Create(po) - assert.Nil(t, err) - } - - // initial health check should already see that there is more than one mon on one node (node0) - _, err := c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - assert.Equal(t, "node0", c.mapping.Node["a"].Name) - assert.Equal(t, "node0", c.mapping.Node["b"].Name) - assert.Equal(t, "mynode0", c.mapping.Node["a"].Hostname) - assert.Equal(t, "mynode0", c.mapping.Node["b"].Hostname) - - // add new node and check if the second mon gets failovered to it - n := &v1.Node{ - Status: v1.NodeStatus{ - Conditions: []v1.NodeCondition{ - { - Type: v1.NodeReady, Status: v1.ConditionTrue, - }, - }, - Addresses: []v1.NodeAddress{ - { - Type: v1.NodeExternalIP, - Address: "2.2.2.2", - }, - }, - }, - } - n.Name = "node2" - clientset.CoreV1().Nodes().Create(n) - - _, err = c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - - // check that mon c exists - logger.Infof("mapping: %+v", c.mapping.Node) - assert.NotNil(t, c.mapping.Node["c"]) - assert.Equal(t, "node2", c.mapping.Node["c"].Name) - - // check if mon b has been deleted - var dlist *apps.DeploymentList - dlist, err = clientset.AppsV1().Deployments(c.Namespace).List(metav1.ListOptions{}) - assert.Nil(t, err) - deleted := true - for _, d := range dlist.Items { - if d.Name == "b" { - deleted = false - } - } - assert.Equal(t, true, deleted, "mon b not failed over or deleted after health check") - - // enable different ceph mon map output - executorNextMons = true - _, err = c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - - // check that nothing has changed - dlist, err = clientset.AppsV1().Deployments(c.Namespace).List(metav1.ListOptions{}) - assert.Nil(t, err) - - for _, d := range dlist.Items { - // both mons should always be on the same node as in this test due to the order - // the mons are processed in the loop - if (d.Name == "rook-ceph-mon-a" && d.Spec.Template.Spec.NodeSelector[v1.LabelHostname] == "node1") || (d.Name != "rook-ceph-mon-c" && d.Spec.Template.Spec.NodeSelector[v1.LabelHostname] == "node2") { - assert.Fail(t, fmt.Sprintf("mon %s shouldn't exist", d.Name)) - } - } -} - -func TestCheckMonsValid(t *testing.T) { - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { - return clienttest.MonInQuorumResponse(), nil - }, - } - clientset := test.New(1) - configDir, _ := ioutil.TempDir("", "") - defer os.RemoveAll(configDir) - context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - } - c := New(context, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - c.waitForStart = false - defer os.RemoveAll(c.context.ConfigDir) - - // add two mons to the mapping on node0 - c.mapping.Node["a"] = &NodeInfo{ - Name: "node0", - Address: "0.0.0.0", - } - c.mapping.Node["b"] = &NodeInfo{ - Name: "node1", - Address: "0.0.0.0", - } - c.maxMonID = 1 - - // add three nodes - for i := 0; i < 3; i++ { - n := &v1.Node{ - Status: v1.NodeStatus{ - Conditions: []v1.NodeCondition{ - { - Type: v1.NodeReady, Status: v1.ConditionTrue, - }, - }, - Addresses: []v1.NodeAddress{ - { - Type: v1.NodeExternalIP, - Address: "0.0.0.0", - }, - }, - }, - } - n.Name = fmt.Sprintf("node%d", i) - clientset.CoreV1().Nodes().Create(n) - } - - _, err := c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - assert.Equal(t, "node0", c.mapping.Node["a"].Name) - assert.Equal(t, "node1", c.mapping.Node["b"].Name) - - // set node0 to contain the not ready status condition, and check that mon.a - // is failovered to be mon.c on node2. note that validity here doesn't include the - // unscheduable flag (for mons already placed on a node). - node0, err := c.context.Clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.Nil(t, err) - node0.Status.Conditions[0].Status = v1.ConditionFalse - _, err = c.context.Clientset.CoreV1().Nodes().Update(node0) - assert.Nil(t, err) - - // add the pods so the getNodesInUse() works correctly - monNames := []string{"a", "b", "c"} - for i := 1; i <= 2; i++ { // 1=b, 2=c - po := c.makeMonPod(testGenMonConfig(monNames[i]), fmt.Sprintf("node%d", i-1)) - _, err = clientset.CoreV1().Pods(c.Namespace).Create(po) - assert.Nil(t, err) - } - - _, err = c.resolveInvalidMonitorPlacement(3) - assert.Nil(t, err) - - assert.Len(t, c.mapping.Node, 2) - logger.Infof("mapping: %+v", c.mapping) - assert.Nil(t, c.mapping.Node["a"]) - // the new mon should always be on the empty node2 - // the failovered mon's name is "rook-ceph-mon-a" - assert.Equal(t, "node2", c.mapping.Node["c"].Name) - assert.Equal(t, "node1", c.mapping.Node["b"].Name) -} - func TestAddRemoveMons(t *testing.T) { var deploymentsUpdated *[]*apps.Deployment updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub() diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index c9df8f58a..e0099ac1b 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -43,6 +43,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) const ( @@ -83,10 +84,18 @@ const ( // default storage request size for ceph monitor pvc // https://docs.ceph.com/docs/master/start/hardware-recommendations/#monitors-and-managers-ceph-mon-and-ceph-mgr cephMonDefaultStorageRequest = "10Gi" + + // canary pod scheduling uses retry loops when cleaning up previous canary + // pods and waiting for kubernetes scheduling to complete. + canaryRetries = 180 + canaryRetryDelaySeconds = 5 ) var ( logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mon") + + // hook for tests to override + scheduleMonitor = realScheduleMonitor ) // Cluster represents the Rook and environment configuration settings needed to set up Ceph mons. @@ -129,7 +138,6 @@ type monConfig struct { // Mapping is mon node and port mapping type Mapping struct { Node map[string]*NodeInfo `json:"node"` - Port map[string]int32 `json:"port"` } // NodeInfo contains name and address of a node @@ -139,6 +147,12 @@ type NodeInfo struct { Address string } +type SchedulingResult struct { + Node *v1.Node + CanaryDeployment string + CanaryPVC string +} + // New creates an instance of a mon cluster func New(context *clusterd.Context, namespace, dataDirHostPath string, network cephv1.NetworkSpec, ownerRef metav1.OwnerReference, csiConfigMutex *sync.Mutex) *Cluster { return &Cluster{ @@ -153,7 +167,6 @@ func New(context *clusterd.Context, namespace, dataDirHostPath string, network c Network: network, mapping: &Mapping{ Node: map[string]*NodeInfo{}, - Port: map[string]int32{}, }, ownerRef: ownerRef, csiConfigMutex: csiConfigMutex, @@ -365,6 +378,136 @@ func resourceName(name string) string { return fmt.Sprintf("%s-%s", AppName, name) } +// scheduleMonitor selects a node for a monitor deployment. +// see startMon() and design/ceph-mon-pv.md for additional details. +func realScheduleMonitor(c *Cluster, mon *monConfig) (SchedulingResult, error) { + // target node decision, and deployment/pvc to cleanup + result := SchedulingResult{ + Node: nil, + CanaryDeployment: "", + CanaryPVC: "", + } + + // build the canary deployment. + d := c.makeDeployment(mon) + d.Name += "-canary" + d.Spec.Template.ObjectMeta.Name += "-canary" + + // the canary and real monitor deployments will mount the same storage. to + // avoid issues with the real deployment, the canary should be careful not + // to modify the storage by instead running an innocuous command. + d.Spec.Template.Spec.InitContainers = []v1.Container{} + d.Spec.Template.Spec.Containers[0].Image = c.rookVersion + d.Spec.Template.Spec.Containers[0].Command = []string{"/tini"} + d.Spec.Template.Spec.Containers[0].Args = []string{"--", "sleep", "3600"} + + // setup affinity settings for pod scheduling + p := cephv1.GetMonPlacement(c.spec.Placement) + c.setPodPlacement(&d.Spec.Template.Spec, p, nil) + + // setup storage on the canary since scheduling will be affected when + // monitors are configured to use persistent volumes. the pvcName is set to + // the non-empty name of the PVC only when the PVC is created as a result of + // this call to the scheduler. + var pvcName string + if c.spec.Mon.VolumeClaimTemplate == nil { + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, + opspec.DaemonVolumesDataHostPath(mon.DataPathMap)...) + } else { + // the pvc that is created here won't be deleted: it will be reattached + // to the real monitor deployment. + pvc, err := c.makeDeploymentPVC(mon) + if err != nil { + return result, fmt.Errorf("sched-mon: failed to make monitor %s pvc. %+v", d.Name, err) + } + + _, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(pvc) + if err == nil { + pvcName = pvc.Name + logger.Infof("sched-mon: created canary monitor %s pvc %s", d.Name, pvc.Name) + } else { + if errors.IsAlreadyExists(err) { + logger.Debugf("sched-mon: creating mon %s pvc %s: already exists.", d.Name, pvc.Name) + } else { + return result, fmt.Errorf("sched-mon: error creating mon %s pvc %s. %+v", d.Name, pvc.Name, err) + } + } + + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, + opspec.DaemonVolumesDataPVC(mon.ResourceName)) + opspec.AddVolumeMountSubPath(&d.Spec.Template.Spec, "ceph-daemon-data") + } + + // caller should arrange for clean-up of the PVC only if it was created + // during this scheduling instance and we encounter an irrecoverable error. + result.CanaryPVC = pvcName + + // spin up the canary deployment. if it exists, delete it first, since if it + // already exists it may have been scheduled with a different crd config. + createdDeployment := false + for i := 0; i < canaryRetries; i++ { + _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d) + if err == nil { + createdDeployment = true + logger.Infof("sched-mon: created canary deployment %s", d.Name) + break + } else if errors.IsAlreadyExists(err) { + if err := k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, d.Name); err != nil { + return result, fmt.Errorf("sched-mon: error deleting canary deployment %s. %+v", d.Name, err) + } + logger.Infof("sched-mon: deleted existing canary deployment %s", d.Name) + time.Sleep(time.Second * canaryRetryDelaySeconds) + } else { + return result, fmt.Errorf("sched-mon: error creating canary monitor deployment %s. %+v", d.Name, err) + } + } + + // failed after retrying + if !createdDeployment { + return result, fmt.Errorf("sched-mon: failed to create canary deployment %s", d.Name) + } + + // caller should arrange for the deployment to be removed + result.CanaryDeployment = d.Name + + // wait for the scheduler to make a placement decision + for i := 0; i < canaryRetries; i++ { + listOptions := metav1.ListOptions{ + LabelSelector: labels.Set(d.Spec.Selector.MatchLabels).String(), + } + + pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(listOptions) + if err != nil { + return result, fmt.Errorf("sched-mon: error listing canary pods %s. %+v", d.Name, err) + } + + if len(pods.Items) == 0 { + logger.Infof("sched-mon: waiting for canary pod creation %s", d.Name) + time.Sleep(time.Second * canaryRetryDelaySeconds) + continue + } + + pod := pods.Items[0] + if pod.Spec.NodeName == "" { + logger.Debugf("sched-mon: monitor %s canary pod %s not yet scheduled", d.Name, pod.Name) + time.Sleep(time.Second * canaryRetryDelaySeconds) + continue + } + + node, err := c.context.Clientset.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + return result, fmt.Errorf("sched-mon: error getting node %s. %+v", pod.Spec.NodeName, err) + } + + result.Node = node + result.CanaryPVC = "" + logger.Infof("sched-mon: canary monitor deployment %s scheduled to %s", d.Name, node.Name) + return result, nil + } + + return result, fmt.Errorf("sched-mon: canary pod scheduling failed retries") +} + func (c *Cluster) initMonIPs(mons []*monConfig) error { for _, m := range mons { if c.Network.IsHost() { @@ -387,129 +530,82 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error { return nil } -// find a suitable node for this monitor. first look for a fault zone that has -// no monitors. only after the nodes with zone labels are considered are the -// unlabeled nodes considered. reasonsing: without more information, _at worst_ -// the unlabled nodes are actually in the set of labeled zones, and at best, -// they are in distinct zones. -func scheduleMonitor(mon *monConfig, nodeZones [][]NodeUsage) *NodeUsage { - // the node choice for this monitor - var nodeChoice *NodeUsage - - // for each zone, in order of preference. unlabeled nodes are last, by - // construction; see Cluster.getNodeMonusage(). - for zi := range nodeZones { - // number of monitors in the zone - zoneMonCount := 0 - - // preferential node choice from this zone for a new mon - var zoneNodeChoice *NodeUsage - - // for each node in this zone - for ni := range nodeZones[zi] { - nodeUsage := &nodeZones[zi][ni] - - // skip invalid nodes. but update the mon count first since - // invalid nodes could still have an assigned mon pod. - zoneMonCount += nodeUsage.MonCount - if !nodeUsage.MonValid { - logger.Infof("schedmon: skip invalid node %s for mon scheduling", - nodeUsage.Node.Name) - continue +func (c *Cluster) assignMons(mons []*monConfig) error { + // when monitors are scheduling below by invoking scheduleMonitor() a canary + // deployment and optional canary PVC are created. in order for the + // anti-affinity rules to be effective, we leave the canary pods in place + // until all of the canaries have been scheduled. only after the + // monitor/node assignment process is complete are the canary deployments + // and pvcs removed here. + canaryCleanup := []SchedulingResult{} + defer func() { + for _, result := range canaryCleanup { + logger.Infof("assignmon: cleaning up canary deployment %s and canary pvc %s", result.CanaryDeployment, result.CanaryPVC) + if result.CanaryDeployment != "" { + if err := k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, result.CanaryDeployment); err != nil { + logger.Infof("assignmon: error deleting canary deployment %s. %+v", result.CanaryDeployment, err) + } } - - // make a "best" choice from this zone. in this case that is the - // node with the least amount of monitors. - if zoneNodeChoice == nil || nodeUsage.MonCount < zoneNodeChoice.MonCount { - logger.Infof("schedmon: considering node %s with mon count %d", - nodeUsage.Node.Name, nodeUsage.MonCount) - zoneNodeChoice = nodeUsage - } - } - - // We identified _a_ valid node in the zone. should we choose it? - if zoneNodeChoice != nil { - if zoneMonCount == 0 { - // this zone has no monitors, which implies that - // zoneNodeChoice will be a node without any monitors - // currently assigned. choose this node. - logger.Infof("schedmon: considering node %s from empty zone", - zoneNodeChoice.Node.Name) - nodeChoice = zoneNodeChoice - break - } else { - // this zone already has a monitor in it. but keep the - // choice as a backup; we may find that all zones already - // have a monitor and still need to make an assignment. - if nodeChoice == nil || zoneNodeChoice.MonCount < nodeChoice.MonCount { - logger.Infof("schedmon: considering node %s with mon count %d", - zoneNodeChoice.Node.Name, zoneNodeChoice.MonCount) - nodeChoice = zoneNodeChoice + if result.CanaryPVC != "" { + var gracePeriod int64 // delete immediately + propagation := metav1.DeletePropagationForeground + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(result.CanaryPVC, options) + if err != nil { + logger.Infof("assignmon: error removing canary monitor %s pvc %s. %+v", result.CanaryDeployment, result.CanaryPVC, err) } } } - } + }() - if nodeChoice != nil { - logger.Infof("schedmon: scheduling mon %s on node %s", - mon.DaemonName, nodeChoice.Node.Name) - } else { - logger.Infof("schedmon: no suitable node found for mon %s", - mon.DaemonName) - } - - return nodeChoice -} - -func (c *Cluster) assignMons(mons []*monConfig) error { - - // retrieve the set of cluster nodes and their monitor usage info - nodeZones, err := c.getNodeMonUsage() - if err != nil { - return fmt.Errorf("failed to get node monitor usage. %+v", err) - } - - // ensure all monitors have a node assignment. note that this isn't - // necessarily optimal: it does not try to move existing monitors which is - // handled by the periodic monitor health checks. + // ensure that all monitors have either (1) a node assignment that will be + // enforced using a node selector, or (2) configuration permits k8s to handle + // scheduling for the monitor. for _, mon := range mons { - // monitor is already assigned to a node. nothing to do + // scheduling for this monitor has already been completed if _, ok := c.mapping.Node[mon.DaemonName]; ok { - logger.Infof("assignmon: mon %s already assigned to a node", mon.DaemonName) + logger.Debugf("assignmon: mon %s already scheduled", mon.DaemonName) continue } - nodeChoice := scheduleMonitor(mon, nodeZones) + // determine a placement for the monitor. note that this scheduling is + // performed even when a node selector is not required. this may be + // non-optimal, but it is convenient to catch some failures early, + // before a decision is stored in the node mapping. + result, err := scheduleMonitor(c, mon) - if nodeChoice == nil { - return fmt.Errorf("assignmon: no valid nodes available for mon placement") - } + // even if an error occurs cleanup may be necessary + canaryCleanup = append(canaryCleanup, result) - // note that we do not need to worry about a false negative here (i.e. a - // node exists that _would_ pass this check) due to the search landing in a - // local minima because if a node had existed with a mon count of zero, - // scheduleMonitor would have chosen it over any node with a positive - // mon count. - if nodeChoice.MonCount > 0 && !c.spec.Mon.AllowMultiplePerNode { - return fmt.Errorf("assignmon: no empty nodes available for mon placement") - } - - // make this decision visible when scheduling the next monitor - nodeChoice.MonCount++ - - logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Node.Name) - - nodeInfo, err := getNodeInfoFromNode(*nodeChoice.Node) if err != nil { - return fmt.Errorf("couldn't get node info from node %s. %+v", - nodeChoice.Node.Name, err) + return fmt.Errorf("assignmon: error scheduling monitor: %+v", err) + } + + nodeChoice := result.Node + if nodeChoice == nil { + return fmt.Errorf("assignmon: could not schedule monitor %s", mon.DaemonName) + } + + // store nil in the node mapping to indicate that an explicit node + // placement is not being made. otherwise, the node choice will map + // directly to a node selector on the monitor pod. + var nodeInfo *NodeInfo = nil + if c.Network.IsHost() || c.spec.Mon.VolumeClaimTemplate == nil { + logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name) + nodeInfo, err = getNodeInfoFromNode(*nodeChoice) + if err != nil { + return fmt.Errorf("assignmon: couldn't get node info for node %s. %+v", + nodeChoice.Name, err) + } + } else { + logger.Infof("assignmon: mon %s placement using native scheduler", mon.DaemonName) } c.mapping.Node[mon.DaemonName] = nodeInfo } - logger.Debug("assignmons: mons have been assigned to nodes") + logger.Debug("assignmons: mons have been scheduled") return nil } @@ -521,7 +617,7 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e // Ensure each of the mons have been created. If already created, it will be a no-op. for i := 0; i < len(mons); i++ { node, _ := c.mapping.Node[mons[i].DaemonName] - err := c.startMon(mons[i], node.Hostname) + err := c.startMon(mons[i], node) if err != nil { return fmt.Errorf("failed to create mon %s. %+v", mons[i].DaemonName, err) } @@ -652,13 +748,42 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error { return nil } -func (c *Cluster) startMon(m *monConfig, hostname string) error { - +// startMon creates or updates a monitor deployment. +// +// The node parameter specifies the node to be used as a node selector on the +// monitor pod. It is the result of scheduling a canary pod: see +// scheduleMonitor() for more details on scheduling. +// +// The node parameter is optional. When the parameter is nil it indicates that +// the pod should not use a node selector, and should instead rely on k8s to +// perform scheduling. +// +// The following outlines the different scenarios that exist and how deployments +// should be configured w.r.t. scheduling and the use of a node selector. +// +// 1) if HostNetworking -> always use node selector. we do not want to change +// the IP address of a monitor as it is wrapped up in the monitor's identity. +// with host networking we use node selector to ensure a stable IP for each +// monitor. see scheduleMonitor() comment for more details. +// +// Note: an important assumption is that HostNetworking setting does not +// change once a cluster is created. +// +// 2) if *not* HostNetworking -> stable IP from service; may avoid node selector +// a) when creating a new deployment +// - if HostPath -> use node selector for storage/node affinity +// - if PVC -> node selector is not required +// +// b) when updating a deployment +// - if HostPath -> leave node selector as is +// - if PVC -> remove node selector, if present +// +func (c *Cluster) startMon(m *monConfig, node *NodeInfo) error { // check if the monitor deployment already exists. if the deployment does // exist, also determine if it using pvc storage. pvcExists := false deploymentExists := false - d := c.makeDeployment(m, hostname) + d := c.makeDeployment(m) existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.Name, metav1.GetOptions{}) if err == nil { deploymentExists = true @@ -683,7 +808,22 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error { logger.Debugf("adding host path volume source to mon deployment %s", d.Name) } + // placement settings from the CRD + p := cephv1.GetMonPlacement(c.spec.Placement) + if deploymentExists { + // the existing deployment may have a node selector. if the cluster + // isn't using host networking and the deployment is using pvc storage, + // then the node selector can be removed. this may happen after + // upgrading the cluster with the k8s scheduling support for monitors. + if c.Network.IsHost() || !pvcExists { + p.PodAffinity = nil + p.PodAntiAffinity = nil + c.setPodPlacement(&d.Spec.Template.Spec, p, + existingDeployment.Spec.Template.Spec.NodeSelector) + } else { + c.setPodPlacement(&d.Spec.Template.Spec, p, nil) + } return c.updateMon(m, d) } @@ -702,6 +842,15 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error { } } + if node == nil { + c.setPodPlacement(&d.Spec.Template.Spec, p, nil) + } else { + p.PodAffinity = nil + p.PodAntiAffinity = nil + c.setPodPlacement(&d.Spec.Template.Spec, p, + map[string]string{v1.LabelHostname: node.Hostname}) + } + logger.Debugf("Starting mon: %+v", d.Name) _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d) if err != nil { diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 43eeaebfd..3c2282074 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -122,7 +122,6 @@ func newCluster(context *clusterd.Context, namespace string, network cephv1.Netw monTimeoutList: map[string]time.Time{}, mapping: &Mapping{ Node: map[string]*NodeInfo{}, - Port: map[string]int32{}, }, ownerRef: metav1.OwnerReference{}, } @@ -241,7 +240,7 @@ func TestSaveMonEndpoints(t *testing.T) { cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, "a=1.2.3.1:6789", cm.Data[EndpointDataKey]) - assert.Equal(t, `{"node":{},"port":{}}`, cm.Data[MappingKey]) + assert.Equal(t, `{"node":{}}`, cm.Data[MappingKey]) assert.Equal(t, "-1", cm.Data[MaxMonIDKey]) // update the config map @@ -252,14 +251,13 @@ func TestSaveMonEndpoints(t *testing.T) { Address: "1.1.1.1", Hostname: "myhost", } - c.mapping.Port["node0"] = int32(12345) err = c.saveMonConfig() assert.Nil(t, err) cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, "a=2.3.4.5:6789", cm.Data[EndpointDataKey]) - assert.Equal(t, `{"node":{"a":{"Name":"node0","Hostname":"myhost","Address":"1.1.1.1"}},"port":{"node0":12345}}`, cm.Data[MappingKey]) + assert.Equal(t, `{"node":{"a":{"Name":"node0","Hostname":"myhost","Address":"1.1.1.1"}}}`, cm.Data[MappingKey]) assert.Equal(t, "2", cm.Data[MaxMonIDKey]) } @@ -344,168 +342,3 @@ func TestMonFoundInQuorum(t *testing.T) { assert.True(t, monFoundInQuorum("c", response)) assert.False(t, monFoundInQuorum("d", response)) } - -// no node choice can be made when there are no nodes -func TestScheduleMonitorEmpty(t *testing.T) { - nodeZones := [][]NodeUsage{} - mon := &monConfig{DaemonName: "a"} - // no zones - assert.Nil(t, scheduleMonitor(mon, nodeZones)) - // 1 zone no mons - nodeZones = append(nodeZones, []NodeUsage{}) - assert.Nil(t, scheduleMonitor(mon, nodeZones)) - // 2 zones no mons - nodeZones = append(nodeZones, []NodeUsage{}) - assert.Nil(t, scheduleMonitor(mon, nodeZones)) -} - -// only valid nodes should be chosen -func TestScheduleMonitorInvalidNodes(t *testing.T) { - // 1 zone with 1 invalid empty node - nodeZones := [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - }, - } - mon := &monConfig{DaemonName: "a"} - assert.Nil(t, scheduleMonitor(mon, nodeZones)) - - // 1 zone with 2 invalid empty node - nodeZones = [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - }, - } - assert.Nil(t, scheduleMonitor(mon, nodeZones)) - - // 2 zone with 2 invalid empty node each - nodeZones = [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - }, - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: false}, - }, - } - assert.Nil(t, scheduleMonitor(mon, nodeZones)) -} - -func TestScheduleMonitor(t *testing.T) { - // 1 zone, 1 valid, empty node -> only one choice - nodeZones := [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - } - mon := &monConfig{DaemonName: "a"} - assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) - - // still the only choice even if not empty - nodeZones = [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, - }, - } - assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) - - // scheduler prefers the node with the least number of mons - nodeZones = [][]NodeUsage{ - { - NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, - }, - } - assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones)) - - // the scheduler prefers nodes with the least number of mons, not zones with - // the leads number of mons (zero mons is a special case: tested below...) - nodeZones = [][]NodeUsage{ - // 24 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 4, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 10, MonValid: true}, - }, - // 10 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 5, MonValid: true}, - }, - } - // choose the node with 4 mons - assert.Equal(t, &nodeZones[0][1], scheduleMonitor(mon, nodeZones)) - - // same as before, the target mon is in the second zone - nodeZones = [][]NodeUsage{ - // 6 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 2, MonValid: true}, - }, - // 10 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 9, MonValid: true}, - }, - } - // choose the node with 1 mon - assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) - - // prefers a zone with zero mons to spread across failure domains - nodeZones = [][]NodeUsage{ - // 0 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - // 1 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, - }, - } - // choose the zone with zero mons - assert.Equal(t, &nodeZones[0][0], scheduleMonitor(mon, nodeZones)) - - // same as before, different zone - nodeZones = [][]NodeUsage{ - // 0 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - // 1 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - } - // choose the zone with zero mons - assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) - - // invalid nodes aren't schedulable, but if they have mons, that factors - // into the decision. in this case it means the zone isn't really empty - nodeZones = [][]NodeUsage{ - // 0 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - // invalid node, but has a mon -> zone is not empty - NodeUsage{Node: &v1.Node{}, MonCount: 1, MonValid: false}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - // 1 mons - { - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - NodeUsage{Node: &v1.Node{}, MonCount: 0, MonValid: true}, - }, - } - // choose the zone with zero mons - assert.Equal(t, &nodeZones[1][0], scheduleMonitor(mon, nodeZones)) -} diff --git a/pkg/operator/ceph/cluster/mon/node.go b/pkg/operator/ceph/cluster/mon/node.go index b8c2a74e3..6b33d28ef 100644 --- a/pkg/operator/ceph/cluster/mon/node.go +++ b/pkg/operator/ceph/cluster/mon/node.go @@ -124,76 +124,3 @@ func getNodeInfoFromNode(n v1.Node) (*NodeInfo, error) { } return nr, nil } - -// Returns a hierarchical representation of the nodes in the cluster organized -// by zone failure domain and annotated with the number of monitor pods assigned -// to each node and if the node is valid for scheduling monitor pods. The -// purpose of this structure is to provide a unified view of information -// required for making monitor scheduling decisions. -func (c *Cluster) getNodeMonUsage() ([][]NodeUsage, error) { - // get all k8s node objects - nodeOptions := metav1.ListOptions{} - nodeOptions.TypeMeta.Kind = "Node" - nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions) - if err != nil { - return nil, err - } - - // get all pod objects labeled as a monitor - podOptions := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)} - pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(podOptions) - if err != nil { - return nil, err - } - - // generate a list of nodes that includes the number of monitor pods - // assigned to each node. this is equivalent to: - // - // SELECT Node, Count(*) FROM Nodes - // JOIN Pods ON Nodes.Hostname == Pods.Hostname - // GROUP BY Node - // - nodeUsages := []NodeUsage{} - for i, node := range nodes.Items { - valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - continue - } - nodeUsage := NodeUsage{Node: &nodes.Items[i], MonCount: 0, MonValid: valid} - for _, pod := range pods.Items { - hostname := pod.Spec.NodeSelector[v1.LabelHostname] - if node.Name == hostname || node.Labels[v1.LabelHostname] == hostname { - nodeUsage.MonCount++ - } - } - nodeUsages = append(nodeUsages, nodeUsage) - } - - // generate a hierarchical representation of the nodes grouped by the name - // of the node's zone failure domain label. the zone label named "" is used - // for nodes without a zone annotation. - nodesByZone := make(map[string][]NodeUsage) - for _, nodeUsage := range nodeUsages { - zone, ok := nodeUsage.Node.Labels["failure-domain.beta.kubernetes.io/zone"] - if !ok { - zone = "" - } - nodesByZone[zone] = append(nodesByZone[zone], nodeUsage) - } - - // compute the final form used for scheduling. this form is an array of - // nodes by zone, with non-labeled nodes appearing in the final zone. this - // organization reflects the preference made by the scheduler of using nodes - // with zone annotations before non-labeled nodes. - res := [][]NodeUsage{} - for zone, nodeUsage := range nodesByZone { - if zone == "" { - res = append(res, nodeUsage) - } else { - res = append([][]NodeUsage{nodeUsage}, res...) - } - } - - return res, nil -} diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index 5b3d78c36..7df406102 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -32,110 +32,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestGetNodeMonUsageValidNode(t *testing.T) { - clientset := test.New(2) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - conditions := v1.NodeCondition{Type: v1.NodeOutOfDisk} - node.Status = v1.NodeStatus{Conditions: []v1.NodeCondition{conditions}} - clientset.CoreV1().Nodes().Update(node) - - // nodes are reported with correct valid flag - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.False(t, nodeZones[0][0].MonValid) - assert.True(t, nodeZones[0][1].MonValid) -} - -func TestGetNodeMonUsageMonCount(t *testing.T) { - clientset := test.New(2) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - // 3 mons on node0 - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - - monIDs := []string{"a", "b", "c"} - for i := 0; i < 3; i++ { - pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name) - _, err = clientset.CoreV1().Pods(c.Namespace).Create(pod) - assert.Nil(t, err) - } - - // 2 mons on node1 - node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - - monIDs = []string{"d", "e"} - for i := 0; i < 2; i++ { - pod := c.makeMonPod(testGenMonConfig(monIDs[i]), node.Name) - _, err = clientset.CoreV1().Pods(c.Namespace).Create(pod) - assert.Nil(t, err) - } - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 1, len(nodeZones)) - if nodeZones[0][0].Node.Name == "node0" { - assert.Equal(t, 3, nodeZones[0][0].MonCount) - assert.Equal(t, 2, nodeZones[0][1].MonCount) - } else { - assert.Equal(t, 2, nodeZones[0][1].MonCount) - assert.Equal(t, 3, nodeZones[0][0].MonCount) - } -} - -func TestTaintedNodes(t *testing.T) { - clientset := test.New(4) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - // mark a node as unschedulable - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Spec.Unschedulable = true - clientset.CoreV1().Nodes().Update(node) - - // taint - node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Spec.Taints = []v1.Taint{ - {Effect: v1.TaintEffectNoSchedule}, - } - clientset.CoreV1().Nodes().Update(node) - - // taint - node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Spec.Taints = []v1.Taint{ - {Effect: v1.TaintEffectPreferNoSchedule}, - } - clientset.CoreV1().Nodes().Update(node) - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 1, len(nodeZones)) - validCount := 0 - for zi := range nodeZones { - for ni := range nodeZones[zi] { - if nodeZones[zi][ni].MonValid { - validCount++ - } - } - } - assert.Equal(t, 1, validCount) -} - func TestNodeAffinity(t *testing.T) { clientset := test.New(4) c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) @@ -160,30 +56,17 @@ func TestNodeAffinity(t *testing.T) { } // label nodes so they appear as not scheduable / invalid - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) + node, _ := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) node.Labels = map[string]string{"label": "foo"} clientset.CoreV1().Nodes().Update(node) - node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) + node, _ = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) node.Labels = map[string]string{"label": "bar"} clientset.CoreV1().Nodes().Update(node) - node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) + node, _ = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) node.Labels = map[string]string{"label": "baz"} clientset.CoreV1().Nodes().Update(node) - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 1, len(nodeZones)) - validCount := 0 - for zi := range nodeZones { - for ni := range nodeZones[zi] { - if nodeZones[zi][ni].MonValid { - validCount++ - } - } - } - assert.Equal(t, 2, validCount) } // this tests can 3 mons with hostnetworking on the same host is rejected @@ -282,22 +165,8 @@ func TestHostNetwork(t *testing.T) { c.Network.HostNetwork = true - nodes := []v1.Node{} - nodeZones, err := c.getNodeMonUsage() - assert.Nil(t, err) - - for zi := range nodeZones { - for _, nodeUsage := range nodeZones[zi] { - if nodeUsage.MonCount == 0 || c.spec.Mon.AllowMultiplePerNode { - nodes = append(nodes, *nodeUsage.Node) - } - } - } - - assert.Equal(t, 3, len(nodes)) - monConfig := testGenMonConfig("c") - pod := c.makeMonPod(monConfig, nodes[2].Name) + pod := c.makeMonPod(monConfig) assert.NotNil(t, pod) assert.Equal(t, true, pod.Spec.HostNetwork) assert.Equal(t, v1.DNSClusterFirstWithHostNet, pod.Spec.DNSPolicy) @@ -308,7 +177,7 @@ func TestHostNetwork(t *testing.T) { assert.Equal(t, "arg not found: --public-bind-addr", message) monConfig.Port = 6790 - pod = c.makeMonPod(monConfig, nodes[2].Name) + pod = c.makeMonPod(monConfig) val, message = extractArgValue(pod.Spec.Containers[0].Args, "--public-addr") assert.Equal(t, "2.4.6.3:6790", val, message) assert.NotNil(t, pod) @@ -399,134 +268,3 @@ func TestTargetMonCount(t *testing.T) { assert.Equal(t, 7, target) logger.Infof(msg) } - -// mon node usage should return no zones if there are no nodes -func TestGetNodeMonUsageNoNodes(t *testing.T) { - clientset := test.New(0) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, len(nodeZones), 0) -} - -// all nodes without zone annotations are in the same zone -func TestGetNodeMonUsageNoZoneLabels(t *testing.T) { - for i := 1; i < 5; i++ { - clientset := test.New(i) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, len(nodeZones), 1) - assert.Equal(t, len(nodeZones[0]), i) - } -} - -// nodes are partitioned into separate zones -func TestGetNodeMonUsageZoneSpread(t *testing.T) { - clientset := test.New(5) - c := New(&clusterd.Context{Clientset: clientset}, "ns", "", cephv1.NetworkSpec{}, metav1.OwnerReference{}, &sync.Mutex{}) - setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "myversion") - - // 1 node labeled -> 1 zone + the rest in the unlabeled zone - node, err := clientset.CoreV1().Nodes().Get("node0", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"} - clientset.CoreV1().Nodes().Update(node) - - // the zones are unlabeled and sort order isn't guaranteed. so instead we - // assert on expected count of zones and collection of zone sizes. - zoneCounts := map[int]int{} - - nodeZones, err := c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 2, len(nodeZones)) - for _, zones := range nodeZones { - zoneSize := len(zones) - if _, ok := zoneCounts[zoneSize]; ok { - zoneCounts[zoneSize]++ - } else { - zoneCounts[zoneSize] = 1 - } - } - assert.Equal(t, map[int]int{1: 1, 4: 1}, zoneCounts) - - // 2 nodes with same label, 3 unlabeled - node, err = clientset.CoreV1().Nodes().Get("node1", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z0"} - clientset.CoreV1().Nodes().Update(node) - - // reset - zoneCounts = map[int]int{} - - nodeZones, err = c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 2, len(nodeZones)) - for _, zones := range nodeZones { - zoneSize := len(zones) - if _, ok := zoneCounts[zoneSize]; ok { - zoneCounts[zoneSize]++ - } else { - zoneCounts[zoneSize] = 1 - } - } - assert.Equal(t, map[int]int{2: 1, 3: 1}, zoneCounts) - - // 2 nodes with same label, 2 with distinct labels, 1 unlabeled - node, err = clientset.CoreV1().Nodes().Get("node2", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"} - clientset.CoreV1().Nodes().Update(node) - - node, err = clientset.CoreV1().Nodes().Get("node3", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z2"} - clientset.CoreV1().Nodes().Update(node) - - // reset - zoneCounts = map[int]int{} - - nodeZones, err = c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 4, len(nodeZones)) - for _, zones := range nodeZones { - zoneSize := len(zones) - if _, ok := zoneCounts[zoneSize]; ok { - zoneCounts[zoneSize]++ - } else { - zoneCounts[zoneSize] = 1 - } - } - assert.Equal(t, map[int]int{2: 1, 1: 3}, zoneCounts) - - // no unlabeled zones - node, err = clientset.CoreV1().Nodes().Get("node4", metav1.GetOptions{}) - assert.NoError(t, err) - assert.NotNil(t, node) - node.Labels = map[string]string{"failure-domain.beta.kubernetes.io/zone": "z1"} - clientset.CoreV1().Nodes().Update(node) - - // reset - zoneCounts = map[int]int{} - - nodeZones, err = c.getNodeMonUsage() - assert.NoError(t, err) - assert.Equal(t, 3, len(nodeZones)) - for _, zones := range nodeZones { - zoneSize := len(zones) - if _, ok := zoneCounts[zoneSize]; ok { - zoneCounts[zoneSize]++ - } else { - zoneCounts[zoneSize] = 1 - } - } - assert.Equal(t, map[int]int{2: 2, 1: 1}, zoneCounts) -} diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index 954f1c6d3..5c9cb50d7 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -48,9 +48,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { }, } k8sutil.SetOwnerRef(&svcDef.ObjectMeta, &c.ownerRef) - if c.Network.IsHost() { - svcDef.Spec.ClusterIP = v1.ClusterIPNone - } // If deploying Nautilus or newer we need a new port for the monitor service if c.ClusterInfo.CephVersion.IsAtLeastNautilus() { diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index f920a25d7..bec0d11d3 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -22,6 +22,7 @@ import ( "path" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rook "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" @@ -52,7 +53,7 @@ func (c *Cluster) getLabels(daemonName string) map[string]string { return labels } -func (c *Cluster) makeDeployment(monConfig *monConfig, hostname string) *apps.Deployment { +func (c *Cluster) makeDeployment(monConfig *monConfig) *apps.Deployment { d := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: monConfig.ResourceName, @@ -65,7 +66,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, hostname string) *apps.De opspec.AddCephVersionLabelToDeployment(c.ClusterInfo.CephVersion, d) k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef) - pod := c.makeMonPod(monConfig, hostname) + pod := c.makeMonPod(monConfig) replicaCount := int32(1) d.Spec = apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -134,7 +135,49 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig) (*v1.PersistentVolumeClaim, er * Pod spec */ -func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { +func (c *Cluster) setPodPlacement(pod *v1.PodSpec, p rook.Placement, nodeSelector map[string]string) { + p.ApplyToPodSpec(pod) + pod.NodeSelector = nodeSelector + + // when a node selector is being used, skip the affinity business below + if nodeSelector != nil { + return + } + + // label selector for monitors used in anti-affinity rules + monAntiAffinity := v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + k8sutil.AppAttr: AppName, + }, + }, + TopologyKey: v1.LabelHostname, + } + + // set monitor pod anti-affinity rules. when monitors should never be + // co-located (e.g. not AllowMultiplePerHost or HostNetworking) then the + // anti-affinity rule is made to be required during scheduling, otherwise it + // is merely a preferred policy. + // + // ApplyToPodSpec ensures that pod.Affinity is non-nil + if pod.Affinity.PodAntiAffinity == nil { + pod.Affinity.PodAntiAffinity = &v1.PodAntiAffinity{} + } + paa := pod.Affinity.PodAntiAffinity + + if c.Network.IsHost() || !c.spec.Mon.AllowMultiplePerNode { + paa.RequiredDuringSchedulingIgnoredDuringExecution = + append(paa.RequiredDuringSchedulingIgnoredDuringExecution, monAntiAffinity) + } else { + paa.PreferredDuringSchedulingIgnoredDuringExecution = + append(paa.PreferredDuringSchedulingIgnoredDuringExecution, v1.WeightedPodAffinityTerm{ + Weight: 50, + PodAffinityTerm: monAntiAffinity, + }) + } +} + +func (c *Cluster) makeMonPod(monConfig *monConfig) *v1.Pod { logger.Debugf("monConfig: %+v", monConfig) podSpec := v1.PodSpec{ InitContainers: []v1.Container{ @@ -145,7 +188,6 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { c.makeMonDaemonContainer(monConfig), }, RestartPolicy: v1.RestartPolicyAlways, - NodeSelector: map[string]string{v1.LabelHostname: hostname}, Volumes: opspec.DaemonVolumesBase(monConfig.DataPathMap, keyringStoreName), HostNetwork: c.Network.IsHost(), } @@ -153,13 +195,6 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, hostname string) *v1.Pod { podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet } - // apply the pod placement if specified in the crd - // remove Pod (anti-)affinity because we have our own placement logic - p := cephv1.GetMonPlacement(c.spec.Placement) - p.PodAffinity = nil - p.PodAntiAffinity = nil - p.ApplyToPodSpec(&podSpec) - pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: monConfig.ResourceName, diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index e876b1f43..bf4ebb3c4 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -21,6 +21,7 @@ import ( "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rook "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" @@ -64,7 +65,7 @@ func testPodSpec(t *testing.T, monID string, pvc bool) { } monConfig := testGenMonConfig(monID) - d := c.makeDeployment(monConfig, "node0") + d := c.makeDeployment(monConfig) assert.NotNil(t, d) if pvc { @@ -145,3 +146,74 @@ func TestDeploymentPVCSpec(t *testing.T) { assert.NoError(t, err) assert.Equal(t, pvc.Spec.Resources.Requests[v1.ResourceStorage], req) } + +func testPodSpecPlacement(t *testing.T, hostNet, allowMulti bool, req, pref int, placement *rook.Placement) { + clientset := testop.New(1) + c := New( + &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"}, + "ns", + "/var/lib/rook", + cephv1.NetworkSpec{HostNetwork: hostNet}, + metav1.OwnerReference{}, + &sync.Mutex{}, + ) + + setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: allowMulti}, "rook/rook:myversion") + monConfig := testGenMonConfig("a") + + if placement != nil { + c.spec.Placement = rook.PlacementSpec{} + c.spec.Placement["mon"] = *placement + } + + d := c.makeDeployment(monConfig) + assert.NotNil(t, d) + + p := cephv1.GetMonPlacement(c.spec.Placement) + c.setPodPlacement(&d.Spec.Template.Spec, p, nil) + + // should have a required anti-affnity and no preferred anti-affinity + assert.Equal(t, + req, + len(d.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution)) + assert.Equal(t, + pref, + len(d.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution)) +} + +func makePlacement() rook.Placement { + return rook.Placement{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + TopologyKey: v1.LabelZoneFailureDomain, + }, + }, + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + PodAffinityTerm: v1.PodAffinityTerm{ + TopologyKey: v1.LabelZoneFailureDomain, + }, + }, + }, + }, + } +} + +func TestPodSpecPlacement(t *testing.T) { + // no placement settings in the crd + testPodSpecPlacement(t, true, true, 1, 0, nil) + testPodSpecPlacement(t, true, false, 1, 0, nil) + testPodSpecPlacement(t, false, true, 0, 1, nil) + testPodSpecPlacement(t, false, false, 1, 0, nil) + + // crd has other preferred and required anti-affinity setting + p := makePlacement() + testPodSpecPlacement(t, true, true, 2, 1, &p) + p = makePlacement() + testPodSpecPlacement(t, true, false, 2, 1, &p) + p = makePlacement() + testPodSpecPlacement(t, false, true, 1, 2, &p) + p = makePlacement() + testPodSpecPlacement(t, false, false, 2, 1, &p) +}