ceph: multiple mgrs have antiaffinity across hosts or zones
If there are multiple mgrs running, the operator will automatically add pod antiaffinity across hosts by default. For stretch clusters, antiaffinity across the stretch failure domain (e.g. zones) will be added. Required antiaffinity will be created unless the test setting allowMultiplePerNode is set to true. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
13 files changed
+49
-27
No files matched your search
@@ -104,9 +104,10 @@ spec:
|
||||
properties:
|
||||
count:
|
||||
type: integer
|
||||
minimum: 1
|
||||
minimum: 0
|
||||
maximum: 2
|
||||
default: 1
|
||||
allowMultiplePerNode:
|
||||
type: boolean
|
||||
modules:
|
||||
type: array
|
||||
items:
|
||||
@@ -1518,8 +1519,10 @@ spec:
|
||||
properties:
|
||||
count:
|
||||
type: integer
|
||||
minimum: 1
|
||||
minimum: 0
|
||||
maximum: 2
|
||||
allowMultiplePerNode:
|
||||
type: boolean
|
||||
modules:
|
||||
items:
|
||||
properties:
|
||||
|
||||
@@ -104,9 +104,11 @@ spec:
|
||||
properties:
|
||||
count:
|
||||
type: integer
|
||||
minimum: 1
|
||||
# 0 will be interpreted as a default of 1 for backward compatibility
|
||||
minimum: 0
|
||||
maximum: 2
|
||||
default: 1
|
||||
allowMultiplePerNode:
|
||||
type: boolean
|
||||
modules:
|
||||
type: array
|
||||
items:
|
||||
|
||||
@@ -88,7 +88,7 @@ spec:
|
||||
properties:
|
||||
count:
|
||||
type: integer
|
||||
minimum: 1
|
||||
minimum: 0
|
||||
maximum: 2
|
||||
modules:
|
||||
items:
|
||||
|
||||
@@ -301,8 +301,9 @@ type StretchClusterZoneSpec struct {
|
||||
|
||||
// MgrSpec represents options to configure a ceph mgr
|
||||
type MgrSpec struct {
|
||||
Count int `json:"count,omitempty"`
|
||||
Modules []Module `json:"modules,omitempty"`
|
||||
Count int `json:"count,omitempty"`
|
||||
AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"`
|
||||
Modules []Module `json:"modules,omitempty"`
|
||||
}
|
||||
|
||||
// Module represents mgr modules that the user wants to enable or disable
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
@@ -69,7 +70,12 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
|
||||
podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig))
|
||||
matchLabels := controller.AppLabels(AppName, c.clusterInfo.Namespace)
|
||||
|
||||
k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, true, matchLabels, nil)
|
||||
// Stretch the mgrs across hosts by default, or across a bigger failure domain for stretch clusters
|
||||
topologyKey := v1.LabelHostname
|
||||
if c.spec.IsStretchCluster() {
|
||||
topologyKey = mon.StretchFailureDomainLabel(c.spec)
|
||||
}
|
||||
k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, !c.spec.Mgr.AllowMultiplePerNode, topologyKey, matchLabels, nil)
|
||||
}
|
||||
|
||||
// If the log collector is enabled we add the side-car container
|
||||
|
||||
@@ -662,7 +662,7 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
|
||||
// setup affinity settings for pod scheduling
|
||||
p := c.getMonPlacement(mon.Zone)
|
||||
p.ApplyToPodSpec(&d.Spec.Template.Spec, true)
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
|
||||
map[string]string{k8sutil.AppAttr: AppName}, nil)
|
||||
|
||||
// setup storage on the canary since scheduling will be affected when
|
||||
@@ -1266,10 +1266,10 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
|
||||
if c.spec.Network.IsHost() || !pvcExists {
|
||||
p.PodAffinity = nil
|
||||
p.PodAntiAffinity = nil
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
|
||||
map[string]string{k8sutil.AppAttr: AppName}, existingDeployment.Spec.Template.Spec.NodeSelector)
|
||||
} else {
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
|
||||
map[string]string{k8sutil.AppAttr: AppName}, nil)
|
||||
}
|
||||
return c.updateMon(m, d)
|
||||
@@ -1291,12 +1291,12 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
|
||||
}
|
||||
|
||||
if schedule == nil || schedule.Zone != "" {
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
|
||||
map[string]string{k8sutil.AppAttr: AppName}, nil)
|
||||
} else {
|
||||
p.PodAffinity = nil
|
||||
p.PodAntiAffinity = nil
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec),
|
||||
k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname,
|
||||
map[string]string{k8sutil.AppAttr: AppName}, map[string]string{v1.LabelHostname: schedule.Hostname})
|
||||
}
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ func (c *Cluster) getLabels(monConfig *monConfig, canary, includeNewLabels bool)
|
||||
}
|
||||
|
||||
func (c *Cluster) stretchFailureDomainName() string {
|
||||
label := c.stretchFailureDomainLabel()
|
||||
label := StretchFailureDomainLabel(c.spec)
|
||||
index := strings.Index(label, "/")
|
||||
if index == -1 {
|
||||
return label
|
||||
@@ -70,9 +70,9 @@ func (c *Cluster) stretchFailureDomainName() string {
|
||||
return label[index+1:]
|
||||
}
|
||||
|
||||
func (c *Cluster) stretchFailureDomainLabel() string {
|
||||
if c.spec.Mon.StretchCluster.FailureDomainLabel != "" {
|
||||
return c.spec.Mon.StretchCluster.FailureDomainLabel
|
||||
func StretchFailureDomainLabel(spec cephv1.ClusterSpec) string {
|
||||
if spec.Mon.StretchCluster.FailureDomainLabel != "" {
|
||||
return spec.Mon.StretchCluster.FailureDomainLabel
|
||||
}
|
||||
// The default topology label is for a zone
|
||||
return corev1.LabelZoneFailureDomainStable
|
||||
@@ -210,7 +210,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*v1.Pod, error)
|
||||
}
|
||||
|
||||
if c.spec.IsStretchCluster() {
|
||||
nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", c.stretchFailureDomainLabel(), monConfig.Zone))
|
||||
nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", StretchFailureDomainLabel(c.spec), monConfig.Zone))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to generate mon %q node affinity", monConfig.DaemonName)
|
||||
}
|
||||
|
||||
@@ -138,7 +138,7 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
|
||||
|
||||
// If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons
|
||||
labels := getLabels(c.store.Name, c.store.Namespace, false)
|
||||
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.clusterSpec.Network.IsHost(), labels, nil)
|
||||
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.clusterSpec.Network.IsHost(), v1.LabelHostname, labels, nil)
|
||||
|
||||
podTemplateSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
|
||||
@@ -311,7 +311,7 @@ func ClusterDaemonEnvVars(image string) []v1.EnvVar {
|
||||
}
|
||||
|
||||
// SetNodeAntiAffinityForPod assign pod anti-affinity when pod should not be co-located
|
||||
func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, labels, nodeSelector map[string]string) {
|
||||
func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, topologyKey string, labels, nodeSelector map[string]string) {
|
||||
pod.NodeSelector = nodeSelector
|
||||
|
||||
// when a node selector is being used, skip the affinity business below
|
||||
@@ -324,7 +324,7 @@ func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, l
|
||||
LabelSelector: &metav1.LabelSelector{
|
||||
MatchLabels: labels,
|
||||
},
|
||||
TopologyKey: v1.LabelHostname,
|
||||
TopologyKey: topologyKey,
|
||||
}
|
||||
|
||||
// Ensures that pod.Affinity is non-nil
|
||||
|
||||
@@ -180,7 +180,7 @@ func testPodSpecPlacement(t *testing.T, requiredDuringScheduling bool, req, pref
|
||||
}
|
||||
|
||||
placement.ApplyToPodSpec(&spec, true)
|
||||
SetNodeAntiAffinityForPod(&spec, requiredDuringScheduling, map[string]string{"app": "mon"}, nil)
|
||||
SetNodeAntiAffinityForPod(&spec, requiredDuringScheduling, v1.LabelHostname, map[string]string{"app": "mon"}, nil)
|
||||
|
||||
// should have a required anti-affinity and no preferred anti-affinity
|
||||
assert.Equal(t,
|
||||
|
||||
@@ -207,7 +207,7 @@ func (h *CephInstaller) Execute(command string, parameters []string, namespace s
|
||||
|
||||
// CreateRookCluster creates rook cluster via kubectl
|
||||
func (h *CephInstaller) CreateRookCluster(namespace, systemNamespace, storeType string, usePVC bool, storageClassName string,
|
||||
mon cephv1.MonSpec, startWithAllNodes bool, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error {
|
||||
mon cephv1.MonSpec, startWithAllNodes, multipleMgrs, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error {
|
||||
|
||||
ctx := context.TODO()
|
||||
dataDirHostPath, err := h.initTestDir(namespace)
|
||||
@@ -250,7 +250,7 @@ osd_pool_default_size = 1
|
||||
}
|
||||
|
||||
logger.Info("Starting Rook Cluster with yaml")
|
||||
settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner}
|
||||
settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, multipleMgrs, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner}
|
||||
rookCluster := h.Manifests.GetRookCluster(settings)
|
||||
logger.Info(rookCluster)
|
||||
maxTry := 3
|
||||
@@ -484,9 +484,10 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st
|
||||
useCrashPruner := rookVersion == VersionMaster
|
||||
|
||||
// Create rook cluster
|
||||
multipleManagers := false
|
||||
err = h.CreateRookCluster(namespace, onamespace, storeType, usePVC, storageClassName,
|
||||
cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes,
|
||||
skipOSDCreation, useCrashPruner, h.CephVersion)
|
||||
multipleManagers, skipOSDCreation, useCrashPruner, h.CephVersion)
|
||||
if err != nil {
|
||||
logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err)
|
||||
return false, err
|
||||
|
||||
@@ -62,6 +62,7 @@ type clusterSettings struct {
|
||||
StoreType string
|
||||
DataDirHostPath string
|
||||
Mons int
|
||||
MultipleMgrs bool
|
||||
RBDMirrorWorkers int
|
||||
UsePVCs bool
|
||||
StorageClassName string
|
||||
@@ -1278,6 +1279,10 @@ func (m *CephManifestsMaster) GetRookCluster(settings *clusterSettings) string {
|
||||
pruner = "daysToRetain: 5"
|
||||
}
|
||||
|
||||
mgrCount := 1
|
||||
if settings.MultipleMgrs {
|
||||
mgrCount = 2
|
||||
}
|
||||
if settings.UsePVCs {
|
||||
return `apiVersion: ceph.rook.io/v1
|
||||
kind: CephCluster
|
||||
@@ -1301,6 +1306,9 @@ spec:
|
||||
allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + `
|
||||
skipUpgradeChecks: false
|
||||
continueUpgradeAfterChecksEvenIfNotHealthy: false
|
||||
mgr:
|
||||
count: ` + strconv.Itoa(mgrCount) + `
|
||||
allowMultiplePerNode: true
|
||||
dashboard:
|
||||
enabled: true
|
||||
network:
|
||||
|
||||
@@ -198,8 +198,9 @@ func (o MCTestOperations) Teardown() {
|
||||
|
||||
func (o MCTestOperations) startCluster(namespace, store string) error {
|
||||
logger.Infof("starting cluster %s", namespace)
|
||||
multipleManagers := true
|
||||
err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName,
|
||||
cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, false, false, installer.NautilusVersion)
|
||||
cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, multipleManagers, false, false, installer.NautilusVersion)
|
||||
if err != nil {
|
||||
o.T().Fail()
|
||||
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
|
||||
|
||||
Reference in new issue
Block a user