From e7331b0191a862e62752e17791dbbbf36b18407c Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Mon, 22 Feb 2021 21:29:11 -0700 Subject: [PATCH] ceph: multiple mgrs have antiaffinity across hosts or zones If there are multiple mgrs running, the operator will automatically add pod antiaffinity across hosts by default. For stretch clusters, antiaffinity across the stretch failure domain (e.g. zones) will be added. Required antiaffinity will be created unless the test setting allowMultiplePerNode is set to true. Signed-off-by: Travis Nielsen --- cluster/charts/rook-ceph/templates/resources.yaml | 9 ++++++--- cluster/examples/kubernetes/ceph/crds.yaml | 6 ++++-- .../examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml | 2 +- pkg/apis/ceph.rook.io/v1/types.go | 5 +++-- pkg/operator/ceph/cluster/mgr/spec.go | 8 +++++++- pkg/operator/ceph/cluster/mon/mon.go | 10 +++++----- pkg/operator/ceph/cluster/mon/spec.go | 10 +++++----- pkg/operator/ceph/object/spec.go | 2 +- pkg/operator/k8sutil/pod.go | 4 ++-- pkg/operator/k8sutil/pod_test.go | 2 +- tests/framework/installer/ceph_installer.go | 7 ++++--- tests/framework/installer/ceph_manifests.go | 8 ++++++++ tests/integration/ceph_multi_cluster_test.go | 3 ++- 13 files changed, 49 insertions(+), 27 deletions(-) diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 101405e66..0217cd29f 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -104,9 +104,10 @@ spec: properties: count: type: integer - minimum: 1 + minimum: 0 maximum: 2 - default: 1 + allowMultiplePerNode: + type: boolean modules: type: array items: @@ -1518,8 +1519,10 @@ spec: properties: count: type: integer - minimum: 1 + minimum: 0 maximum: 2 + allowMultiplePerNode: + type: boolean modules: items: properties: diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index fd739243b..3b2859340 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -104,9 +104,11 @@ spec: properties: count: type: integer - minimum: 1 + # 0 will be interpreted as a default of 1 for backward compatibility + minimum: 0 maximum: 2 - default: 1 + allowMultiplePerNode: + type: boolean modules: type: array items: diff --git a/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml b/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml index 17e99af42..261f08848 100644 --- a/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml +++ b/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml @@ -88,7 +88,7 @@ spec: properties: count: type: integer - minimum: 1 + minimum: 0 maximum: 2 modules: items: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index a65ec232f..2ecb389d3 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -301,8 +301,9 @@ type StretchClusterZoneSpec struct { // MgrSpec represents options to configure a ceph mgr type MgrSpec struct { - Count int `json:"count,omitempty"` - Modules []Module `json:"modules,omitempty"` + Count int `json:"count,omitempty"` + AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"` + Modules []Module `json:"modules,omitempty"` } // Module represents mgr modules that the user wants to enable or disable diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 8656df258..8d47e5d24 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -24,6 +24,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" @@ -69,7 +70,12 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig)) matchLabels := controller.AppLabels(AppName, c.clusterInfo.Namespace) - k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, true, matchLabels, nil) + // Stretch the mgrs across hosts by default, or across a bigger failure domain for stretch clusters + topologyKey := v1.LabelHostname + if c.spec.IsStretchCluster() { + topologyKey = mon.StretchFailureDomainLabel(c.spec) + } + k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, !c.spec.Mgr.AllowMultiplePerNode, topologyKey, matchLabels, nil) } // If the log collector is enabled we add the side-car container diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index b7ac6a076..db60f3683 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -662,7 +662,7 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { // setup affinity settings for pod scheduling p := c.getMonPlacement(mon.Zone) p.ApplyToPodSpec(&d.Spec.Template.Spec, true) - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, nil) // setup storage on the canary since scheduling will be affected when @@ -1266,10 +1266,10 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { if c.spec.Network.IsHost() || !pvcExists { p.PodAffinity = nil p.PodAntiAffinity = nil - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, existingDeployment.Spec.Template.Spec.NodeSelector) } else { - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, nil) } return c.updateMon(m, d) @@ -1291,12 +1291,12 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { } if schedule == nil || schedule.Zone != "" { - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, nil) } else { p.PodAffinity = nil p.PodAntiAffinity = nil - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, map[string]string{v1.LabelHostname: schedule.Hostname}) } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 092c497e8..64e503505 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -62,7 +62,7 @@ func (c *Cluster) getLabels(monConfig *monConfig, canary, includeNewLabels bool) } func (c *Cluster) stretchFailureDomainName() string { - label := c.stretchFailureDomainLabel() + label := StretchFailureDomainLabel(c.spec) index := strings.Index(label, "/") if index == -1 { return label @@ -70,9 +70,9 @@ func (c *Cluster) stretchFailureDomainName() string { return label[index+1:] } -func (c *Cluster) stretchFailureDomainLabel() string { - if c.spec.Mon.StretchCluster.FailureDomainLabel != "" { - return c.spec.Mon.StretchCluster.FailureDomainLabel +func StretchFailureDomainLabel(spec cephv1.ClusterSpec) string { + if spec.Mon.StretchCluster.FailureDomainLabel != "" { + return spec.Mon.StretchCluster.FailureDomainLabel } // The default topology label is for a zone return corev1.LabelZoneFailureDomainStable @@ -210,7 +210,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*v1.Pod, error) } if c.spec.IsStretchCluster() { - nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", c.stretchFailureDomainLabel(), monConfig.Zone)) + nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", StretchFailureDomainLabel(c.spec), monConfig.Zone)) if err != nil { return nil, errors.Wrapf(err, "failed to generate mon %q node affinity", monConfig.DaemonName) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 1fb7620cc..e62faa528 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -138,7 +138,7 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec // If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons labels := getLabels(c.store.Name, c.store.Namespace, false) - k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.clusterSpec.Network.IsHost(), labels, nil) + k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.clusterSpec.Network.IsHost(), v1.LabelHostname, labels, nil) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index 39a0ab816..0f0774d32 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -311,7 +311,7 @@ func ClusterDaemonEnvVars(image string) []v1.EnvVar { } // SetNodeAntiAffinityForPod assign pod anti-affinity when pod should not be co-located -func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, labels, nodeSelector map[string]string) { +func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, topologyKey string, labels, nodeSelector map[string]string) { pod.NodeSelector = nodeSelector // when a node selector is being used, skip the affinity business below @@ -324,7 +324,7 @@ func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, l LabelSelector: &metav1.LabelSelector{ MatchLabels: labels, }, - TopologyKey: v1.LabelHostname, + TopologyKey: topologyKey, } // Ensures that pod.Affinity is non-nil diff --git a/pkg/operator/k8sutil/pod_test.go b/pkg/operator/k8sutil/pod_test.go index eece64bb1..f744aad06 100644 --- a/pkg/operator/k8sutil/pod_test.go +++ b/pkg/operator/k8sutil/pod_test.go @@ -180,7 +180,7 @@ func testPodSpecPlacement(t *testing.T, requiredDuringScheduling bool, req, pref } placement.ApplyToPodSpec(&spec, true) - SetNodeAntiAffinityForPod(&spec, requiredDuringScheduling, map[string]string{"app": "mon"}, nil) + SetNodeAntiAffinityForPod(&spec, requiredDuringScheduling, v1.LabelHostname, map[string]string{"app": "mon"}, nil) // should have a required anti-affinity and no preferred anti-affinity assert.Equal(t, diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 94db3c22c..9dabffe74 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -207,7 +207,7 @@ func (h *CephInstaller) Execute(command string, parameters []string, namespace s // CreateRookCluster creates rook cluster via kubectl func (h *CephInstaller) CreateRookCluster(namespace, systemNamespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes bool, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error { + mon cephv1.MonSpec, startWithAllNodes, multipleMgrs, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error { ctx := context.TODO() dataDirHostPath, err := h.initTestDir(namespace) @@ -250,7 +250,7 @@ osd_pool_default_size = 1 } logger.Info("Starting Rook Cluster with yaml") - settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner} + settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, multipleMgrs, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner} rookCluster := h.Manifests.GetRookCluster(settings) logger.Info(rookCluster) maxTry := 3 @@ -484,9 +484,10 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st useCrashPruner := rookVersion == VersionMaster // Create rook cluster + multipleManagers := false err = h.CreateRookCluster(namespace, onamespace, storeType, usePVC, storageClassName, cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes, - skipOSDCreation, useCrashPruner, h.CephVersion) + multipleManagers, skipOSDCreation, useCrashPruner, h.CephVersion) if err != nil { logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err) return false, err diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index a249d471c..2ef0213ef 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -62,6 +62,7 @@ type clusterSettings struct { StoreType string DataDirHostPath string Mons int + MultipleMgrs bool RBDMirrorWorkers int UsePVCs bool StorageClassName string @@ -1278,6 +1279,10 @@ func (m *CephManifestsMaster) GetRookCluster(settings *clusterSettings) string { pruner = "daysToRetain: 5" } + mgrCount := 1 + if settings.MultipleMgrs { + mgrCount = 2 + } if settings.UsePVCs { return `apiVersion: ceph.rook.io/v1 kind: CephCluster @@ -1301,6 +1306,9 @@ spec: allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` skipUpgradeChecks: false continueUpgradeAfterChecksEvenIfNotHealthy: false + mgr: + count: ` + strconv.Itoa(mgrCount) + ` + allowMultiplePerNode: true dashboard: enabled: true network: diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index d1f6373c5..7e5e2af2f 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -198,8 +198,9 @@ func (o MCTestOperations) Teardown() { func (o MCTestOperations) startCluster(namespace, store string) error { logger.Infof("starting cluster %s", namespace) + multipleManagers := true err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName, - cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, false, false, installer.NautilusVersion) + cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, multipleManagers, false, false, installer.NautilusVersion) if err != nil { o.T().Fail() o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)