From eebb367e4257000f5b24563aa7b01ef78c8370b7 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 3 Dec 2020 23:45:51 -0700 Subject: [PATCH] ceph: allow arbiter to have separate placement from other mons The arbiter may have unique requirements for placement settings. Thus, if the arbiter placement is specified in the cluster CR, that placement will be applied to the arbiter mon and the other mons will retain their placement from the other mon or all sections of the placement. Signed-off-by: Travis Nielsen --- Documentation/ceph-cluster-crd.md | 8 ++- .../kubernetes/ceph/cluster-stretched.yaml | 8 +++ pkg/apis/ceph.rook.io/v1/keys.go | 1 + pkg/apis/ceph.rook.io/v1/placement.go | 7 +++ pkg/operator/ceph/cluster/mon/mon.go | 44 +++++++++++++--- pkg/operator/ceph/cluster/mon/mon_test.go | 51 +++++++++++++++++++ 6 files changed, 112 insertions(+), 7 deletions(-) diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index d00db80d5..d3af12ae4 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -519,7 +519,13 @@ When other keys are set, `all` will be merged together with the specific compone ### Placement Configuration Settings -Placement configuration for the cluster services. It includes the following keys: `mgr`, `mon`, `osd`, `cleanup`, and `all`. Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes). +Placement configuration for the cluster services. It includes the following keys: `mgr`, `mon`, `arbiter`, `osd`, `cleanup`, and `all`. +Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes). + +In stretch clusters, if the `arbiter` placement is specified, that placement will only be applied to the arbiter. +Neither will the `arbiter` placement be merged with the `all` placement to allow the arbiter to be fully independent of other daemon placement. +The remaining mons will still use the `mon` and/or `all` sections. + **NOTE:** Placement of OSD pods is controlled using the [Storage Class Device Set](#storage-class-device-sets), not the general `placement` configuration. diff --git a/cluster/examples/kubernetes/ceph/cluster-stretched.yaml b/cluster/examples/kubernetes/ceph/cluster-stretched.yaml index 312c81183..1cb5a3d73 100644 --- a/cluster/examples/kubernetes/ceph/cluster-stretched.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-stretched.yaml @@ -50,6 +50,14 @@ spec: deviceFilter: "" # OSD placement is expected to include the non-arbiter zones placement: + # The arbiter mon can have its own placement settings that will be different from the mons. + # If the arbiter section is not included in the placement, the arbiter will use the same placement + # settings as other mons. In this example, the arbiter has a toleration to run on a master node. + arbiter: + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule osd: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/pkg/apis/ceph.rook.io/v1/keys.go b/pkg/apis/ceph.rook.io/v1/keys.go index fd82cc94c..6d3f58de5 100644 --- a/pkg/apis/ceph.rook.io/v1/keys.go +++ b/pkg/apis/ceph.rook.io/v1/keys.go @@ -22,6 +22,7 @@ import ( const ( KeyMon rook.KeyType = "mon" + KeyMonArbiter rook.KeyType = "arbiter" KeyMgr rook.KeyType = "mgr" KeyOSDPrepare rook.KeyType = "prepareosd" KeyOSD rook.KeyType = "osd" diff --git a/pkg/apis/ceph.rook.io/v1/placement.go b/pkg/apis/ceph.rook.io/v1/placement.go index 41c479210..698dd544b 100644 --- a/pkg/apis/ceph.rook.io/v1/placement.go +++ b/pkg/apis/ceph.rook.io/v1/placement.go @@ -29,6 +29,13 @@ func GetMonPlacement(p rookv1.PlacementSpec) rookv1.Placement { return p.All().Merge(p[KeyMon]) } +// GetArbiterPlacement returns the placement for the arbiter MON service +func GetArbiterPlacement(p rookv1.PlacementSpec) rookv1.Placement { + // If the mon is the arbiter in a stretch cluster and its placement is specified, return it + // without merging with the "all" placement so it can be handled separately from all other daemons + return p[KeyMonArbiter] +} + // GetOSDPlacement returns the placement for the OSD service func GetOSDPlacement(p rookv1.PlacementSpec) rookv1.Placement { return p.All().Merge(p[KeyOSD]) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 7a7fd6d6f..03c919fef 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "strconv" "strings" "sync" @@ -32,6 +33,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" @@ -324,14 +326,24 @@ func (c *Cluster) configureStretchCluster(mons []*monConfig) error { return nil } -func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error { - var arbiterZone string +func (c *Cluster) getArbiterZone() string { for _, zone := range c.spec.Mon.StretchCluster.Zones { if zone.Arbiter { - arbiterZone = zone.Name - break + return zone.Name } } + return "" +} + +func (c *Cluster) isArbiterZone(zone string) bool { + if !c.spec.IsStretchCluster() { + return false + } + return c.getArbiterZone() == zone +} + +func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error { + arbiterZone := c.getArbiterZone() // Set the location for each mon domainName := c.stretchFailureDomainName() @@ -553,7 +565,7 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { d.Spec.Template.Spec.Containers[0].LivenessProbe = nil // setup affinity settings for pod scheduling - p := cephv1.GetMonPlacement(c.spec.Placement) + p := c.getMonPlacement(mon.Zone) k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, p, requiredDuringScheduling(&c.spec), map[string]string{k8sutil.AppAttr: AppName}, nil) @@ -617,6 +629,22 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { return d, nil } +// GetMonPlacement returns the placement for the MON service +func (c *Cluster) getMonPlacement(zone string) rookv1.Placement { + // If the mon is the arbiter in a stretch cluster and its placement is specified, return it + // without merging with the "all" placement so it can be handled separately from all other daemons + if c.isArbiterZone(zone) { + p := cephv1.GetArbiterPlacement(c.spec.Placement) + noPlacement := rookv1.Placement{} + if !reflect.DeepEqual(p, noPlacement) { + // If the arbiter placement was specified, go ahead and use it. + return p + } + } + // If not the arbiter, or the arbiter placement is not specified, fall back to the same placement used for other mons + return cephv1.GetMonPlacement(c.spec.Placement) +} + func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingResult, error) { ctx := context.TODO() // target node decision, and deployment/pvc to cleanup @@ -1073,7 +1101,11 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { } // placement settings from the CRD - p := cephv1.GetMonPlacement(c.spec.Placement) + var zone string + if schedule != nil { + zone = schedule.Zone + } + p := c.getMonPlacement(zone) if deploymentExists { // the existing deployment may have a node selector. if the cluster diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 3b1a74ee8..016322ce0 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -31,6 +31,7 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" @@ -468,3 +469,53 @@ func TestStretchMonVolumeClaimTemplate(t *testing.T) { }) } } + +func TestArbiterPlacement(t *testing.T) { + placement := rookv1.Placement{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpExists, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + } + c := &Cluster{spec: cephv1.ClusterSpec{ + Mon: cephv1.MonSpec{ + StretchCluster: &cephv1.StretchClusterSpec{ + Zones: []cephv1.StretchClusterZoneSpec{ + {Name: "a", Arbiter: true}, + {Name: "b"}, + {Name: "c"}, + }, + }, + }, + }} + + c.spec.Placement = rookv1.PlacementSpec{} + c.spec.Placement[cephv1.KeyMonArbiter] = placement + + // No placement is found if not requesting the arbiter placement + result := c.getMonPlacement("c") + assert.Equal(t, rookv1.Placement{}, result) + + // Placement is found if requesting the arbiter + result = c.getMonPlacement("a") + assert.Equal(t, placement, result) + + // Arbiter and all mons have the same placement if no arbiter placement is specified + c.spec.Placement = rookv1.PlacementSpec{} + c.spec.Placement[cephv1.KeyMon] = placement + result = c.getMonPlacement("a") + assert.Equal(t, placement, result) + result = c.getMonPlacement("c") + assert.Equal(t, placement, result) +}