ceph: multiple mgrs have antiaffinity across hosts or zones

If there are multiple mgrs running, the operator will automatically
add pod antiaffinity across hosts by default. For stretch clusters,
antiaffinity across the stretch failure domain (e.g. zones)
will be added. Required antiaffinity will be created unless the
test setting allowMultiplePerNode is set to true.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2021-03-10 08:37:17 -07:00
1 parent 142ba63b6a
commit e7331b0191
13 files changed
+49 -27

No files matched your search

@@ -104,9 +104,10 @@ spec:
properties:
count:
type: integer
minimum: 1
minimum: 0
maximum: 2
default: 1
allowMultiplePerNode:
type: boolean
modules:
type: array
items:
@@ -1518,8 +1519,10 @@ spec:
properties:
count:
type: integer
minimum: 1
minimum: 0
maximum: 2
allowMultiplePerNode:
type: boolean
modules:
items:
properties:
+4 -2
View File
@@ -104,9 +104,11 @@ spec:
properties:
count:
type: integer
minimum: 1
# 0 will be interpreted as a default of 1 for backward compatibility
minimum: 0
maximum: 2
default: 1
allowMultiplePerNode:
type: boolean
modules:
type: array
items:
@@ -88,7 +88,7 @@ spec:
properties:
count:
type: integer
minimum: 1
minimum: 0
maximum: 2
modules:
items:
+3 -2
View File
@@ -301,8 +301,9 @@ type StretchClusterZoneSpec struct {
// MgrSpec represents options to configure a ceph mgr
type MgrSpec struct {
Count int `json:"count,omitempty"`
Modules []Module `json:"modules,omitempty"`
Count int `json:"count,omitempty"`
AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"`
Modules []Module `json:"modules,omitempty"`
}
// Module represents mgr modules that the user wants to enable or disable
+7 -1
View File
@@ -24,6 +24,7 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
@@ -69,7 +70,12 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig))
matchLabels := controller.AppLabels(AppName, c.clusterInfo.Namespace)
k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, true, matchLabels, nil)
// Stretch the mgrs across hosts by default, or across a bigger failure domain for stretch clusters
topologyKey := v1.LabelHostname
if c.spec.IsStretchCluster() {
topologyKey = mon.StretchFailureDomainLabel(c.spec)
}
k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, !c.spec.Mgr.AllowMultiplePerNode, topologyKey, matchLabels, nil)
}
// If the log collector is enabled we add the side-car container
+5 -5
View File
@@ -662,7 +662,7 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
// setup affinity settings for pod scheduling
p := c.getMonPlacement(mon.Zone)
p.ApplyToPodSpec(&d.Spec.Template.Spec, true)
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
map[string]string{k8sutil.AppAttr: AppName}, nil)
// setup storage on the canary since scheduling will be affected when
@@ -1266,10 +1266,10 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
if c.spec.Network.IsHost() || !pvcExists {
p.PodAffinity = nil
p.PodAntiAffinity = nil
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
map[string]string{k8sutil.AppAttr: AppName}, existingDeployment.Spec.Template.Spec.NodeSelector)
} else {
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
map[string]string{k8sutil.AppAttr: AppName}, nil)
}
return c.updateMon(m, d)
@@ -1291,12 +1291,12 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
}
if schedule == nil || schedule.Zone != "" {
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
map[string]string{k8sutil.AppAttr: AppName}, nil)
} else {
p.PodAffinity = nil
p.PodAntiAffinity = nil
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
map[string]string{k8sutil.AppAttr: AppName}, map[string]string{v1.LabelHostname: schedule.Hostname})
}
+5 -5
View File
@@ -62,7 +62,7 @@ func (c *Cluster) getLabels(monConfig *monConfig, canary, includeNewLabels bool)
}
func (c *Cluster) stretchFailureDomainName() string {
label := c.stretchFailureDomainLabel()
label := StretchFailureDomainLabel(c.spec)
index := strings.Index(label, "/")
if index == -1 {
return label
@@ -70,9 +70,9 @@ func (c *Cluster) stretchFailureDomainName() string {
return label[index+1:]
}
func (c *Cluster) stretchFailureDomainLabel() string {
if c.spec.Mon.StretchCluster.FailureDomainLabel != "" {
return c.spec.Mon.StretchCluster.FailureDomainLabel
func StretchFailureDomainLabel(spec cephv1.ClusterSpec) string {
if spec.Mon.StretchCluster.FailureDomainLabel != "" {
return spec.Mon.StretchCluster.FailureDomainLabel
}
// The default topology label is for a zone
return corev1.LabelZoneFailureDomainStable
@@ -210,7 +210,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*v1.Pod, error)
}
if c.spec.IsStretchCluster() {
nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", c.stretchFailureDomainLabel(), monConfig.Zone))
nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", StretchFailureDomainLabel(c.spec), monConfig.Zone))
if err != nil {
return nil, errors.Wrapf(err, "failed to generate mon %q node affinity", monConfig.DaemonName)
}
+1 -1
View File
@@ -138,7 +138,7 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
// If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons
labels := getLabels(c.store.Name, c.store.Namespace, false)
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.clusterSpec.Network.IsHost(), labels, nil)
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.clusterSpec.Network.IsHost(), v1.LabelHostname, labels, nil)
podTemplateSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
+2 -2
View File
@@ -311,7 +311,7 @@ func ClusterDaemonEnvVars(image string) []v1.EnvVar {
}
// SetNodeAntiAffinityForPod assign pod anti-affinity when pod should not be co-located
func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, labels, nodeSelector map[string]string) {
func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, topologyKey string, labels, nodeSelector map[string]string) {
pod.NodeSelector = nodeSelector
// when a node selector is being used, skip the affinity business below
@@ -324,7 +324,7 @@ func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, l
LabelSelector: &metav1.LabelSelector{
MatchLabels: labels,
},
TopologyKey: v1.LabelHostname,
TopologyKey: topologyKey,
}
// Ensures that pod.Affinity is non-nil
+1 -1
View File
@@ -180,7 +180,7 @@ func testPodSpecPlacement(t *testing.T, requiredDuringScheduling bool, req, pref
}
placement.ApplyToPodSpec(&spec, true)
SetNodeAntiAffinityForPod(&spec, requiredDuringScheduling, map[string]string{"app": "mon"}, nil)
SetNodeAntiAffinityForPod(&spec, requiredDuringScheduling, v1.LabelHostname, map[string]string{"app": "mon"}, nil)
// should have a required anti-affinity and no preferred anti-affinity
assert.Equal(t,
+4 -3
View File
@@ -207,7 +207,7 @@ func (h *CephInstaller) Execute(command string, parameters []string, namespace s
// CreateRookCluster creates rook cluster via kubectl
func (h *CephInstaller) CreateRookCluster(namespace, systemNamespace, storeType string, usePVC bool, storageClassName string,
mon cephv1.MonSpec, startWithAllNodes bool, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error {
mon cephv1.MonSpec, startWithAllNodes, multipleMgrs, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error {
ctx := context.TODO()
dataDirHostPath, err := h.initTestDir(namespace)
@@ -250,7 +250,7 @@ osd_pool_default_size = 1
}
logger.Info("Starting Rook Cluster with yaml")
settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner}
settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, multipleMgrs, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner}
rookCluster := h.Manifests.GetRookCluster(settings)
logger.Info(rookCluster)
maxTry := 3
@@ -484,9 +484,10 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st
useCrashPruner := rookVersion == VersionMaster
// Create rook cluster
multipleManagers := false
err = h.CreateRookCluster(namespace, onamespace, storeType, usePVC, storageClassName,
cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes,
skipOSDCreation, useCrashPruner, h.CephVersion)
multipleManagers, skipOSDCreation, useCrashPruner, h.CephVersion)
if err != nil {
logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err)
return false, err
@@ -62,6 +62,7 @@ type clusterSettings struct {
StoreType string
DataDirHostPath string
Mons int
MultipleMgrs bool
RBDMirrorWorkers int
UsePVCs bool
StorageClassName string
@@ -1278,6 +1279,10 @@ func (m *CephManifestsMaster) GetRookCluster(settings *clusterSettings) string {
pruner = "daysToRetain: 5"
}
mgrCount := 1
if settings.MultipleMgrs {
mgrCount = 2
}
if settings.UsePVCs {
return `apiVersion: ceph.rook.io/v1
kind: CephCluster
@@ -1301,6 +1306,9 @@ spec:
allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + `
skipUpgradeChecks: false
continueUpgradeAfterChecksEvenIfNotHealthy: false
mgr:
count: ` + strconv.Itoa(mgrCount) + `
allowMultiplePerNode: true
dashboard:
enabled: true
network:
+2 -1
View File
@@ -198,8 +198,9 @@ func (o MCTestOperations) Teardown() {
func (o MCTestOperations) startCluster(namespace, store string) error {
logger.Infof("starting cluster %s", namespace)
multipleManagers := true
err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName,
cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, false, false, installer.NautilusVersion)
cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, multipleManagers, false, false, installer.NautilusVersion)
if err != nil {
o.T().Fail()
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)