ceph: allow arbiter to have separate placement from other mons
The arbiter may have unique requirements for placement settings. Thus, if the arbiter placement is specified in the cluster CR, that placement will be applied to the arbiter mon and the other mons will retain their placement from the other mon or all sections of the placement. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
6 files changed
+112
-7
No files matched your search
@@ -519,7 +519,13 @@ When other keys are set, `all` will be merged together with the specific compone
|
||||
|
||||
### Placement Configuration Settings
|
||||
|
||||
Placement configuration for the cluster services. It includes the following keys: `mgr`, `mon`, `osd`, `cleanup`, and `all`. Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes).
|
||||
Placement configuration for the cluster services. It includes the following keys: `mgr`, `mon`, `arbiter`, `osd`, `cleanup`, and `all`.
|
||||
Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes).
|
||||
|
||||
In stretch clusters, if the `arbiter` placement is specified, that placement will only be applied to the arbiter.
|
||||
Neither will the `arbiter` placement be merged with the `all` placement to allow the arbiter to be fully independent of other daemon placement.
|
||||
The remaining mons will still use the `mon` and/or `all` sections.
|
||||
|
||||
|
||||
**NOTE:** Placement of OSD pods is controlled using the [Storage Class Device Set](#storage-class-device-sets), not the general `placement` configuration.
|
||||
|
||||
|
||||
@@ -50,6 +50,14 @@ spec:
|
||||
deviceFilter: ""
|
||||
# OSD placement is expected to include the non-arbiter zones
|
||||
placement:
|
||||
# The arbiter mon can have its own placement settings that will be different from the mons.
|
||||
# If the arbiter section is not included in the placement, the arbiter will use the same placement
|
||||
# settings as other mons. In this example, the arbiter has a toleration to run on a master node.
|
||||
arbiter:
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/master
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
osd:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
|
||||
const (
|
||||
KeyMon rook.KeyType = "mon"
|
||||
KeyMonArbiter rook.KeyType = "arbiter"
|
||||
KeyMgr rook.KeyType = "mgr"
|
||||
KeyOSDPrepare rook.KeyType = "prepareosd"
|
||||
KeyOSD rook.KeyType = "osd"
|
||||
|
||||
@@ -29,6 +29,13 @@ func GetMonPlacement(p rookv1.PlacementSpec) rookv1.Placement {
|
||||
return p.All().Merge(p[KeyMon])
|
||||
}
|
||||
|
||||
// GetArbiterPlacement returns the placement for the arbiter MON service
|
||||
func GetArbiterPlacement(p rookv1.PlacementSpec) rookv1.Placement {
|
||||
// If the mon is the arbiter in a stretch cluster and its placement is specified, return it
|
||||
// without merging with the "all" placement so it can be handled separately from all other daemons
|
||||
return p[KeyMonArbiter]
|
||||
}
|
||||
|
||||
// GetOSDPlacement returns the placement for the OSD service
|
||||
func GetOSDPlacement(p rookv1.PlacementSpec) rookv1.Placement {
|
||||
return p.All().Merge(p[KeyOSD])
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -32,6 +33,7 @@ import (
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
@@ -324,14 +326,24 @@ func (c *Cluster) configureStretchCluster(mons []*monConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error {
|
||||
var arbiterZone string
|
||||
func (c *Cluster) getArbiterZone() string {
|
||||
for _, zone := range c.spec.Mon.StretchCluster.Zones {
|
||||
if zone.Arbiter {
|
||||
arbiterZone = zone.Name
|
||||
break
|
||||
return zone.Name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (c *Cluster) isArbiterZone(zone string) bool {
|
||||
if !c.spec.IsStretchCluster() {
|
||||
return false
|
||||
}
|
||||
return c.getArbiterZone() == zone
|
||||
}
|
||||
|
||||
func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error {
|
||||
arbiterZone := c.getArbiterZone()
|
||||
|
||||
// Set the location for each mon
|
||||
domainName := c.stretchFailureDomainName()
|
||||
@@ -553,7 +565,7 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
|
||||
d.Spec.Template.Spec.Containers[0].LivenessProbe = nil
|
||||
|
||||
// setup affinity settings for pod scheduling
|
||||
p := cephv1.GetMonPlacement(c.spec.Placement)
|
||||
p := c.getMonPlacement(mon.Zone)
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, p, requiredDuringScheduling(&c.spec),
|
||||
map[string]string{k8sutil.AppAttr: AppName}, nil)
|
||||
|
||||
@@ -617,6 +629,22 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// GetMonPlacement returns the placement for the MON service
|
||||
func (c *Cluster) getMonPlacement(zone string) rookv1.Placement {
|
||||
// If the mon is the arbiter in a stretch cluster and its placement is specified, return it
|
||||
// without merging with the "all" placement so it can be handled separately from all other daemons
|
||||
if c.isArbiterZone(zone) {
|
||||
p := cephv1.GetArbiterPlacement(c.spec.Placement)
|
||||
noPlacement := rookv1.Placement{}
|
||||
if !reflect.DeepEqual(p, noPlacement) {
|
||||
// If the arbiter placement was specified, go ahead and use it.
|
||||
return p
|
||||
}
|
||||
}
|
||||
// If not the arbiter, or the arbiter placement is not specified, fall back to the same placement used for other mons
|
||||
return cephv1.GetMonPlacement(c.spec.Placement)
|
||||
}
|
||||
|
||||
func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingResult, error) {
|
||||
ctx := context.TODO()
|
||||
// target node decision, and deployment/pvc to cleanup
|
||||
@@ -1073,7 +1101,11 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
|
||||
}
|
||||
|
||||
// placement settings from the CRD
|
||||
p := cephv1.GetMonPlacement(c.spec.Placement)
|
||||
var zone string
|
||||
if schedule != nil {
|
||||
zone = schedule.Zone
|
||||
}
|
||||
p := c.getMonPlacement(zone)
|
||||
|
||||
if deploymentExists {
|
||||
// the existing deployment may have a node selector. if the cluster
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
clienttest "github.com/rook/rook/pkg/daemon/ceph/client/test"
|
||||
@@ -468,3 +469,53 @@ func TestStretchMonVolumeClaimTemplate(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestArbiterPlacement(t *testing.T) {
|
||||
placement := rookv1.Placement{
|
||||
NodeAffinity: &v1.NodeAffinity{
|
||||
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
|
||||
NodeSelectorTerms: []v1.NodeSelectorTerm{
|
||||
{
|
||||
MatchExpressions: []v1.NodeSelectorRequirement{
|
||||
{
|
||||
Key: "foo",
|
||||
Operator: v1.NodeSelectorOpExists,
|
||||
Values: []string{"bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
c := &Cluster{spec: cephv1.ClusterSpec{
|
||||
Mon: cephv1.MonSpec{
|
||||
StretchCluster: &cephv1.StretchClusterSpec{
|
||||
Zones: []cephv1.StretchClusterZoneSpec{
|
||||
{Name: "a", Arbiter: true},
|
||||
{Name: "b"},
|
||||
{Name: "c"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}}
|
||||
|
||||
c.spec.Placement = rookv1.PlacementSpec{}
|
||||
c.spec.Placement[cephv1.KeyMonArbiter] = placement
|
||||
|
||||
// No placement is found if not requesting the arbiter placement
|
||||
result := c.getMonPlacement("c")
|
||||
assert.Equal(t, rookv1.Placement{}, result)
|
||||
|
||||
// Placement is found if requesting the arbiter
|
||||
result = c.getMonPlacement("a")
|
||||
assert.Equal(t, placement, result)
|
||||
|
||||
// Arbiter and all mons have the same placement if no arbiter placement is specified
|
||||
c.spec.Placement = rookv1.PlacementSpec{}
|
||||
c.spec.Placement[cephv1.KeyMon] = placement
|
||||
result = c.getMonPlacement("a")
|
||||
assert.Equal(t, placement, result)
|
||||
result = c.getMonPlacement("c")
|
||||
assert.Equal(t, placement, result)
|
||||
}
|
||||
Reference in new issue
Block a user