ceph: allow arbiter to have separate placement from other mons

The arbiter may have unique requirements for placement settings. Thus, if the arbiter
placement is specified in the cluster CR, that placement will be applied to the arbiter
mon and the other mons will retain their placement from the other mon or all
sections of the placement.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2020-12-07 10:58:20 -07:00
1 parent 6a00b4d296
commit eebb367e42
6 files changed
+112 -7

No files matched your search

+7 -1
View File
@@ -519,7 +519,13 @@ When other keys are set, `all` will be merged together with the specific compone
### Placement Configuration Settings
Placement configuration for the cluster services. It includes the following keys: `mgr`, `mon`, `osd`, `cleanup`, and `all`. Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes).
Placement configuration for the cluster services. It includes the following keys: `mgr`, `mon`, `arbiter`, `osd`, `cleanup`, and `all`.
Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes).
In stretch clusters, if the `arbiter` placement is specified, that placement will only be applied to the arbiter.
Neither will the `arbiter` placement be merged with the `all` placement to allow the arbiter to be fully independent of other daemon placement.
The remaining mons will still use the `mon` and/or `all` sections.
**NOTE:** Placement of OSD pods is controlled using the [Storage Class Device Set](#storage-class-device-sets), not the general `placement` configuration.
@@ -50,6 +50,14 @@ spec:
deviceFilter: ""
# OSD placement is expected to include the non-arbiter zones
placement:
# The arbiter mon can have its own placement settings that will be different from the mons.
# If the arbiter section is not included in the placement, the arbiter will use the same placement
# settings as other mons. In this example, the arbiter has a toleration to run on a master node.
arbiter:
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
osd:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
+1
View File
@@ -22,6 +22,7 @@ import (
const (
KeyMon rook.KeyType = "mon"
KeyMonArbiter rook.KeyType = "arbiter"
KeyMgr rook.KeyType = "mgr"
KeyOSDPrepare rook.KeyType = "prepareosd"
KeyOSD rook.KeyType = "osd"
+7
View File
@@ -29,6 +29,13 @@ func GetMonPlacement(p rookv1.PlacementSpec) rookv1.Placement {
return p.All().Merge(p[KeyMon])
}
// GetArbiterPlacement returns the placement for the arbiter MON service
func GetArbiterPlacement(p rookv1.PlacementSpec) rookv1.Placement {
// If the mon is the arbiter in a stretch cluster and its placement is specified, return it
// without merging with the "all" placement so it can be handled separately from all other daemons
return p[KeyMonArbiter]
}
// GetOSDPlacement returns the placement for the OSD service
func GetOSDPlacement(p rookv1.PlacementSpec) rookv1.Placement {
return p.All().Merge(p[KeyOSD])
+38 -6
View File
@@ -23,6 +23,7 @@ import (
"context"
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
"sync"
@@ -32,6 +33,7 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
@@ -324,14 +326,24 @@ func (c *Cluster) configureStretchCluster(mons []*monConfig) error {
return nil
}
func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error {
var arbiterZone string
func (c *Cluster) getArbiterZone() string {
for _, zone := range c.spec.Mon.StretchCluster.Zones {
if zone.Arbiter {
arbiterZone = zone.Name
break
return zone.Name
}
}
return ""
}
func (c *Cluster) isArbiterZone(zone string) bool {
if !c.spec.IsStretchCluster() {
return false
}
return c.getArbiterZone() == zone
}
func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error {
arbiterZone := c.getArbiterZone()
// Set the location for each mon
domainName := c.stretchFailureDomainName()
@@ -553,7 +565,7 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
d.Spec.Template.Spec.Containers[0].LivenessProbe = nil
// setup affinity settings for pod scheduling
p := cephv1.GetMonPlacement(c.spec.Placement)
p := c.getMonPlacement(mon.Zone)
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, p, requiredDuringScheduling(&c.spec),
map[string]string{k8sutil.AppAttr: AppName}, nil)
@@ -617,6 +629,22 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
return d, nil
}
// GetMonPlacement returns the placement for the MON service
func (c *Cluster) getMonPlacement(zone string) rookv1.Placement {
// If the mon is the arbiter in a stretch cluster and its placement is specified, return it
// without merging with the "all" placement so it can be handled separately from all other daemons
if c.isArbiterZone(zone) {
p := cephv1.GetArbiterPlacement(c.spec.Placement)
noPlacement := rookv1.Placement{}
if !reflect.DeepEqual(p, noPlacement) {
// If the arbiter placement was specified, go ahead and use it.
return p
}
}
// If not the arbiter, or the arbiter placement is not specified, fall back to the same placement used for other mons
return cephv1.GetMonPlacement(c.spec.Placement)
}
func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingResult, error) {
ctx := context.TODO()
// target node decision, and deployment/pvc to cleanup
@@ -1073,7 +1101,11 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
}
// placement settings from the CRD
p := cephv1.GetMonPlacement(c.spec.Placement)
var zone string
if schedule != nil {
zone = schedule.Zone
}
p := c.getMonPlacement(zone)
if deploymentExists {
// the existing deployment may have a node selector. if the cluster
+51
View File
@@ -31,6 +31,7 @@ import (
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test"
@@ -468,3 +469,53 @@ func TestStretchMonVolumeClaimTemplate(t *testing.T) {
})
}
}
func TestArbiterPlacement(t *testing.T) {
placement := rookv1.Placement{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpExists,
Values: []string{"bar"},
},
},
},
},
},
},
}
c := &Cluster{spec: cephv1.ClusterSpec{
Mon: cephv1.MonSpec{
StretchCluster: &cephv1.StretchClusterSpec{
Zones: []cephv1.StretchClusterZoneSpec{
{Name: "a", Arbiter: true},
{Name: "b"},
{Name: "c"},
},
},
},
}}
c.spec.Placement = rookv1.PlacementSpec{}
c.spec.Placement[cephv1.KeyMonArbiter] = placement
// No placement is found if not requesting the arbiter placement
result := c.getMonPlacement("c")
assert.Equal(t, rookv1.Placement{}, result)
// Placement is found if requesting the arbiter
result = c.getMonPlacement("a")
assert.Equal(t, placement, result)
// Arbiter and all mons have the same placement if no arbiter placement is specified
c.spec.Placement = rookv1.PlacementSpec{}
c.spec.Placement[cephv1.KeyMon] = placement
result = c.getMonPlacement("a")
assert.Equal(t, placement, result)
result = c.getMonPlacement("c")
assert.Equal(t, placement, result)
}