diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 101405e66..0217cd29f 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -104,9 +104,10 @@ spec: properties: count: type: integer - minimum: 1 + minimum: 0 maximum: 2 - default: 1 + allowMultiplePerNode: + type: boolean modules: type: array items: @@ -1518,8 +1519,10 @@ spec: properties: count: type: integer - minimum: 1 + minimum: 0 maximum: 2 + allowMultiplePerNode: + type: boolean modules: items: properties: diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index fd739243b..3b2859340 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -104,9 +104,11 @@ spec: properties: count: type: integer - minimum: 1 + # 0 will be interpreted as a default of 1 for backward compatibility + minimum: 0 maximum: 2 - default: 1 + allowMultiplePerNode: + type: boolean modules: type: array items: diff --git a/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml b/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml index 17e99af42..261f08848 100644 --- a/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml +++ b/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml @@ -88,7 +88,7 @@ spec: properties: count: type: integer - minimum: 1 + minimum: 0 maximum: 2 modules: items: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index a65ec232f..2ecb389d3 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -301,8 +301,9 @@ type StretchClusterZoneSpec struct { // MgrSpec represents options to configure a ceph mgr type MgrSpec struct { - Count int `json:"count,omitempty"` - Modules []Module `json:"modules,omitempty"` + Count int `json:"count,omitempty"` + AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"` + Modules []Module `json:"modules,omitempty"` } // Module represents mgr modules that the user wants to enable or disable diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 8656df258..8d47e5d24 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -24,6 +24,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" @@ -69,7 +70,12 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig)) matchLabels := controller.AppLabels(AppName, c.clusterInfo.Namespace) - k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, true, matchLabels, nil) + // Stretch the mgrs across hosts by default, or across a bigger failure domain for stretch clusters + topologyKey := v1.LabelHostname + if c.spec.IsStretchCluster() { + topologyKey = mon.StretchFailureDomainLabel(c.spec) + } + k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, !c.spec.Mgr.AllowMultiplePerNode, topologyKey, matchLabels, nil) } // If the log collector is enabled we add the side-car container diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index b7ac6a076..db60f3683 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -662,7 +662,7 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { // setup affinity settings for pod scheduling p := c.getMonPlacement(mon.Zone) p.ApplyToPodSpec(&d.Spec.Template.Spec, true) - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, nil) // setup storage on the canary since scheduling will be affected when @@ -1266,10 +1266,10 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { if c.spec.Network.IsHost() || !pvcExists { p.PodAffinity = nil p.PodAntiAffinity = nil - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, existingDeployment.Spec.Template.Spec.NodeSelector) } else { - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, nil) } return c.updateMon(m, d) @@ -1291,12 +1291,12 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { } if schedule == nil || schedule.Zone != "" { - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, nil) } else { p.PodAffinity = nil p.PodAntiAffinity = nil - k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), + k8sutil.SetNodeAntiAffinityForPod(&d.Spec.Template.Spec, requiredDuringScheduling(&c.spec), v1.LabelHostname, map[string]string{k8sutil.AppAttr: AppName}, map[string]string{v1.LabelHostname: schedule.Hostname}) } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 092c497e8..64e503505 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -62,7 +62,7 @@ func (c *Cluster) getLabels(monConfig *monConfig, canary, includeNewLabels bool) } func (c *Cluster) stretchFailureDomainName() string { - label := c.stretchFailureDomainLabel() + label := StretchFailureDomainLabel(c.spec) index := strings.Index(label, "/") if index == -1 { return label @@ -70,9 +70,9 @@ func (c *Cluster) stretchFailureDomainName() string { return label[index+1:] } -func (c *Cluster) stretchFailureDomainLabel() string { - if c.spec.Mon.StretchCluster.FailureDomainLabel != "" { - return c.spec.Mon.StretchCluster.FailureDomainLabel +func StretchFailureDomainLabel(spec cephv1.ClusterSpec) string { + if spec.Mon.StretchCluster.FailureDomainLabel != "" { + return spec.Mon.StretchCluster.FailureDomainLabel } // The default topology label is for a zone return corev1.LabelZoneFailureDomainStable @@ -210,7 +210,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*v1.Pod, error) } if c.spec.IsStretchCluster() { - nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", c.stretchFailureDomainLabel(), monConfig.Zone)) + nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", StretchFailureDomainLabel(c.spec), monConfig.Zone)) if err != nil { return nil, errors.Wrapf(err, "failed to generate mon %q node affinity", monConfig.DaemonName) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 1fb7620cc..e62faa528 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -138,7 +138,7 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec // If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons labels := getLabels(c.store.Name, c.store.Namespace, false) - k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.clusterSpec.Network.IsHost(), labels, nil) + k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.clusterSpec.Network.IsHost(), v1.LabelHostname, labels, nil) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index 39a0ab816..0f0774d32 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -311,7 +311,7 @@ func ClusterDaemonEnvVars(image string) []v1.EnvVar { } // SetNodeAntiAffinityForPod assign pod anti-affinity when pod should not be co-located -func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, labels, nodeSelector map[string]string) { +func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, topologyKey string, labels, nodeSelector map[string]string) { pod.NodeSelector = nodeSelector // when a node selector is being used, skip the affinity business below @@ -324,7 +324,7 @@ func SetNodeAntiAffinityForPod(pod *v1.PodSpec, requiredDuringScheduling bool, l LabelSelector: &metav1.LabelSelector{ MatchLabels: labels, }, - TopologyKey: v1.LabelHostname, + TopologyKey: topologyKey, } // Ensures that pod.Affinity is non-nil diff --git a/pkg/operator/k8sutil/pod_test.go b/pkg/operator/k8sutil/pod_test.go index eece64bb1..f744aad06 100644 --- a/pkg/operator/k8sutil/pod_test.go +++ b/pkg/operator/k8sutil/pod_test.go @@ -180,7 +180,7 @@ func testPodSpecPlacement(t *testing.T, requiredDuringScheduling bool, req, pref } placement.ApplyToPodSpec(&spec, true) - SetNodeAntiAffinityForPod(&spec, requiredDuringScheduling, map[string]string{"app": "mon"}, nil) + SetNodeAntiAffinityForPod(&spec, requiredDuringScheduling, v1.LabelHostname, map[string]string{"app": "mon"}, nil) // should have a required anti-affinity and no preferred anti-affinity assert.Equal(t, diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 94db3c22c..9dabffe74 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -207,7 +207,7 @@ func (h *CephInstaller) Execute(command string, parameters []string, namespace s // CreateRookCluster creates rook cluster via kubectl func (h *CephInstaller) CreateRookCluster(namespace, systemNamespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes bool, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error { + mon cephv1.MonSpec, startWithAllNodes, multipleMgrs, skipOSDCreation bool, useCrashPruner bool, cephVersion cephv1.CephVersionSpec) error { ctx := context.TODO() dataDirHostPath, err := h.initTestDir(namespace) @@ -250,7 +250,7 @@ osd_pool_default_size = 1 } logger.Info("Starting Rook Cluster with yaml") - settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner} + settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, multipleMgrs, 0, usePVC, storageClassName, skipOSDCreation, cephVersion, useCrashPruner} rookCluster := h.Manifests.GetRookCluster(settings) logger.Info(rookCluster) maxTry := 3 @@ -484,9 +484,10 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st useCrashPruner := rookVersion == VersionMaster // Create rook cluster + multipleManagers := false err = h.CreateRookCluster(namespace, onamespace, storeType, usePVC, storageClassName, cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes, - skipOSDCreation, useCrashPruner, h.CephVersion) + multipleManagers, skipOSDCreation, useCrashPruner, h.CephVersion) if err != nil { logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err) return false, err diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index a249d471c..2ef0213ef 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -62,6 +62,7 @@ type clusterSettings struct { StoreType string DataDirHostPath string Mons int + MultipleMgrs bool RBDMirrorWorkers int UsePVCs bool StorageClassName string @@ -1278,6 +1279,10 @@ func (m *CephManifestsMaster) GetRookCluster(settings *clusterSettings) string { pruner = "daysToRetain: 5" } + mgrCount := 1 + if settings.MultipleMgrs { + mgrCount = 2 + } if settings.UsePVCs { return `apiVersion: ceph.rook.io/v1 kind: CephCluster @@ -1301,6 +1306,9 @@ spec: allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + ` skipUpgradeChecks: false continueUpgradeAfterChecksEvenIfNotHealthy: false + mgr: + count: ` + strconv.Itoa(mgrCount) + ` + allowMultiplePerNode: true dashboard: enabled: true network: diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index d1f6373c5..7e5e2af2f 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -198,8 +198,9 @@ func (o MCTestOperations) Teardown() { func (o MCTestOperations) startCluster(namespace, store string) error { logger.Infof("starting cluster %s", namespace) + multipleManagers := true err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName, - cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, false, false, installer.NautilusVersion) + cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, multipleManagers, false, false, installer.NautilusVersion) if err != nil { o.T().Fail() o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)