diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index d00db80d5..d3af12ae4 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -519,7 +519,13 @@ When other keys are set, `all` will be merged together with the specific compone ### Placement Configuration Settings -Placement configuration for the cluster services. It includes the following keys: `mgr`, `mon`, `osd`, `cleanup`, and `all`. Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes). +Placement configuration for the cluster services. It includes the following keys: `mgr`, `mon`, `arbiter`, `osd`, `cleanup`, and `all`. +Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes). + +In stretch clusters, if the `arbiter` placement is specified, that placement will only be applied to the arbiter. +Neither will the `arbiter` placement be merged with the `all` placement to allow the arbiter to be fully independent of other daemon placement. +The remaining mons will still use the `mon` and/or `all` sections. + **NOTE:** Placement of OSD pods is controlled using the [Storage Class Device Set](#storage-class-device-sets), not the general `placement` configuration. diff --git a/cluster/examples/kubernetes/ceph/cluster-stretched.yaml b/cluster/examples/kubernetes/ceph/cluster-stretched.yaml index 312c81183..1cb5a3d73 100644 --- a/cluster/examples/kubernetes/ceph/cluster-stretched.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-stretched.yaml @@ -50,6 +50,14 @@ spec: deviceFilter: "" # OSD placement is expected to include the non-arbiter zones placement: + # The arbiter mon can have its own placement settings that will be different from the mons. + # If the arbiter section is not included in the placement, the arbiter will use the same placement + # settings as other mons. In this example, the arbiter has a toleration to run on a master node. + arbiter: + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule osd: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/pkg/apis/ceph.rook.io/v1/keys.go b/pkg/apis/ceph.rook.io/v1/keys.go index fd82cc94c..6d3f58de5 100644 --- a/pkg/apis/ceph.rook.io/v1/keys.go +++ b/pkg/apis/ceph.rook.io/v1/keys.go @@ -22,6 +22,7 @@ import ( const ( KeyMon rook.KeyType = "mon" + KeyMonArbiter rook.KeyType = "arbiter" KeyMgr rook.KeyType = "mgr" KeyOSDPrepare rook.KeyType = "prepareosd" KeyOSD rook.KeyType = "osd" diff --git a/pkg/apis/ceph.rook.io/v1/placement.go b/pkg/apis/ceph.rook.io/v1/placement.go index 41c479210..698dd544b 100644 --- a/pkg/apis/ceph.rook.io/v1/placement.go +++ b/pkg/apis/ceph.rook.io/v1/placement.go @@ -29,6 +29,13 @@ func GetMonPlacement(p rookv1.PlacementSpec) rookv1.Placement { return p.All().Merge(p[KeyMon]) } +// GetArbiterPlacement returns the placement for the arbiter MON service +func GetArbiterPlacement(p rookv1.PlacementSpec) rookv1.Placement { + // If the mon is the arbiter in a stretch cluster and its placement is specified, return it + // without merging with the "all" placement so it can be handled separately from all other daemons + return p[KeyMonArbiter] +} + // GetOSDPlacement returns the placement for the OSD service func GetOSDPlacement(p rookv1.PlacementSpec) rookv1.Placement { return p.All().Merge(p[KeyOSD]) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 7a7fd6d6f..03c919fef 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "strconv" "strings" "sync" @@ -32,6 +33,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" @@ -324,14 +326,24 @@ func (c *Cluster) configureStretchCluster(mons []*monConfig) error { return nil } -func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error { - var arbiterZone string +func (c *Cluster) getArbiterZone() string { for _, zone := range c.spec.Mon.StretchCluster.Zones { if zone.Arbiter { - arbiterZone = zone.Name - break + return zone.Name } } + return "" +} + +func (c *Cluster) isArbiterZone(zone string) bool { + if !c.spec.IsStretchCluster() { + return false + } + return c.getArbiterZone() == zone +} + +func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error { + arbiterZone := c.getArbiterZone() // Set the location for each mon domainName := c.stretchFailureDomainName() @@ -553,7 +565,7 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { d.Spec.Template.Spec.Containers[0].LivenessProbe = nil // setup affinity settings for pod scheduling - p := cephv1.GetMonPlacement(c.spec.Placement) + p := c.getMonPlacement(mon.Zone) k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, p, requiredDuringScheduling(&c.spec), map[string]string{k8sutil.AppAttr: AppName}, nil) @@ -617,6 +629,22 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { return d, nil } +// GetMonPlacement returns the placement for the MON service +func (c *Cluster) getMonPlacement(zone string) rookv1.Placement { + // If the mon is the arbiter in a stretch cluster and its placement is specified, return it + // without merging with the "all" placement so it can be handled separately from all other daemons + if c.isArbiterZone(zone) { + p := cephv1.GetArbiterPlacement(c.spec.Placement) + noPlacement := rookv1.Placement{} + if !reflect.DeepEqual(p, noPlacement) { + // If the arbiter placement was specified, go ahead and use it. + return p + } + } + // If not the arbiter, or the arbiter placement is not specified, fall back to the same placement used for other mons + return cephv1.GetMonPlacement(c.spec.Placement) +} + func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingResult, error) { ctx := context.TODO() // target node decision, and deployment/pvc to cleanup @@ -1073,7 +1101,11 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { } // placement settings from the CRD - p := cephv1.GetMonPlacement(c.spec.Placement) + var zone string + if schedule != nil { + zone = schedule.Zone + } + p := c.getMonPlacement(zone) if deploymentExists { // the existing deployment may have a node selector. if the cluster diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 3b1a74ee8..016322ce0 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -31,6 +31,7 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test" @@ -468,3 +469,53 @@ func TestStretchMonVolumeClaimTemplate(t *testing.T) { }) } } + +func TestArbiterPlacement(t *testing.T) { + placement := rookv1.Placement{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpExists, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + } + c := &Cluster{spec: cephv1.ClusterSpec{ + Mon: cephv1.MonSpec{ + StretchCluster: &cephv1.StretchClusterSpec{ + Zones: []cephv1.StretchClusterZoneSpec{ + {Name: "a", Arbiter: true}, + {Name: "b"}, + {Name: "c"}, + }, + }, + }, + }} + + c.spec.Placement = rookv1.PlacementSpec{} + c.spec.Placement[cephv1.KeyMonArbiter] = placement + + // No placement is found if not requesting the arbiter placement + result := c.getMonPlacement("c") + assert.Equal(t, rookv1.Placement{}, result) + + // Placement is found if requesting the arbiter + result = c.getMonPlacement("a") + assert.Equal(t, placement, result) + + // Arbiter and all mons have the same placement if no arbiter placement is specified + c.spec.Placement = rookv1.PlacementSpec{} + c.spec.Placement[cephv1.KeyMon] = placement + result = c.getMonPlacement("a") + assert.Equal(t, placement, result) + result = c.getMonPlacement("c") + assert.Equal(t, placement, result) +}