From c4ed09c3e6b89425eeacdc9d77b45de20091dcd6 Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Wed, 5 Jul 2023 16:29:04 +0530 Subject: [PATCH] mon: add failure domain label for zonal topology Signed-off-by: Deepika Upadhyay (cherry picked from commit 764f6597c0b7f6cc28dc1f57c1c9a582e47c6a38) --- Documentation/CRDs/specification.md | 11 +++++++++++ pkg/apis/ceph.rook.io/v1/cluster.go | 11 ++++++++++- pkg/apis/ceph.rook.io/v1/types.go | 2 ++ pkg/operator/ceph/cluster/mgr/spec.go | 6 +++--- pkg/operator/ceph/cluster/mon/mon.go | 12 ++++++------ pkg/operator/ceph/cluster/mon/spec.go | 16 ++++++++++------ 6 files changed, 42 insertions(+), 16 deletions(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index fe6765ba8..f3ab41b25 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7490,6 +7490,17 @@ bool +failureDomainLabel
+ +string + + + +(Optional) + + + + zones
diff --git a/pkg/apis/ceph.rook.io/v1/cluster.go b/pkg/apis/ceph.rook.io/v1/cluster.go index 68df441b2..05b4d7515 100644 --- a/pkg/apis/ceph.rook.io/v1/cluster.go +++ b/pkg/apis/ceph.rook.io/v1/cluster.go @@ -50,12 +50,21 @@ func (c *ClusterSpec) IsStretchCluster() bool { return c.Mon.StretchCluster != nil && len(c.Mon.StretchCluster.Zones) > 0 } +func (c *ClusterSpec) ZonesRequired() bool { + return c.IsStretchCluster() || len(c.Mon.Zones) > 0 +} + func (c *CephCluster) ValidateCreate() (admission.Warnings, error) { + if c.Spec.ZonesRequired() { + if len(c.Spec.Mon.Zones) < c.Spec.Mon.Count { + return nil, errors.New("Not enough zones available for mons, please specify more zones") + } + } logger.Infof("validate create cephcluster %q", c.ObjectMeta.Name) //If external mode enabled, then check if other fields are empty if c.Spec.External.Enable { if !reflect.DeepEqual(c.Spec.Mon, MonSpec{}) || c.Spec.Dashboard != (DashboardSpec{}) || !reflect.DeepEqual(c.Spec.Monitoring, (MonitoringSpec{})) || c.Spec.DisruptionManagement != (DisruptionManagementSpec{}) || len(c.Spec.Mgr.Modules) > 0 || len(c.Spec.Network.Provider) > 0 || len(c.Spec.Network.Selectors) > 0 { - return errors.New("invalid create : external mode enabled cannot have mon,dashboard,monitoring,network,disruptionManagement,storage fields in CR") + return nil, errors.New("invalid create : external mode enabled cannot have mon,dashboard,monitoring,network,disruptionManagement,storage fields in CR") } } return nil, nil diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 4811facf3..55abc034f 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -530,6 +530,8 @@ type MonSpec struct { // AllowMultiplePerNode determines if we can run multiple monitors on the same node (not recommended) // +optional AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"` + // +optional + FailureDomainLabel string `json:"failureDomainLabel,omitempty"` // Zones are specified when we want to provide zonal awareness to mons // +optional Zones []MonZoneSpec `json:"zones,omitempty"` diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index b8422af21..bf616e5ee 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -76,10 +76,10 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) matchLabels := controller.AppLabels(AppName, c.clusterInfo.Namespace) podSpec.Spec.Volumes = append(podSpec.Spec.Volumes, mon.CephSecretVolume()) - // Stretch the mgrs across hosts by default, or across a bigger failure domain for stretch clusters + // Stretch the mgrs across hosts by default, or across a bigger failure domain for when zones are required like in case of stretched cluster topologyKey := v1.LabelHostname - if c.spec.IsStretchCluster() { - topologyKey = mon.StretchFailureDomainLabel(c.spec) + if c.spec.ZonesRequired() { + topologyKey = mon.GetFailureDomainLabel(c.spec) } k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, !c.spec.Mgr.AllowMultiplePerNode, topologyKey, matchLabels, nil) } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 38ebd12fd..c4ebc9a9b 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -314,7 +314,7 @@ func (c *Cluster) configureStretchCluster(mons []*monConfig) error { } // Create the default crush rule for stretch clusters, that by default will also apply to all pools - if err := cephclient.CreateDefaultStretchCrushRule(c.context, c.ClusterInfo, &c.spec, c.stretchFailureDomainName()); err != nil { + if err := cephclient.CreateDefaultStretchCrushRule(c.context, c.ClusterInfo, &c.spec, c.getFailureDomainName()); err != nil { return errors.Wrap(err, "failed to create default stretch rule") } @@ -367,7 +367,7 @@ func (c *Cluster) ConfigureArbiter() error { // Wait for the CRUSH map to have at least two zones // The timeout is relatively short since the operator will requeue the reconcile // and try again at a higher level if not yet found - failureDomain := c.stretchFailureDomainName() + failureDomain := c.getFailureDomainName() logger.Infof("enabling stretch mode... waiting for two failure domains of type %q to be found in the CRUSH map after OSD initialization", failureDomain) pollInterval := 5 * time.Second totalWaitTime := 2 * time.Minute @@ -387,7 +387,7 @@ func (c *Cluster) ConfigureArbiter() error { } func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) { - failureDomain := c.stretchFailureDomainName() + failureDomain := c.getFailureDomainName() if checkOSDPods { // Wait for the OSD pods to be running @@ -596,7 +596,7 @@ func (c *Cluster) newMonConfig(monID int, zone string) *monConfig { } func (c *Cluster) findAvailableZone(mons []*monConfig) (string, error) { - if !c.spec.IsStretchCluster() && !(len(c.spec.Mon.Zones) > 0) { + if !c.spec.ZonesRequired() { return "", nil } @@ -926,7 +926,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error { } else { logger.Infof("mon %q placement using native scheduler", mon.DaemonName) } - if (len(c.spec.Mon.Zones) > 0) || c.spec.IsStretchCluster() { + if c.spec.ZonesRequired() { if schedule == nil { schedule = &controller.MonScheduleInfo{} } @@ -952,7 +952,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error { func (c *Cluster) monVolumeClaimTemplate(mon *monConfig) *v1.PersistentVolumeClaim { - if c.spec.IsStretchCluster() || len(c.spec.Mon.Zones) > 0 { + if c.spec.ZonesRequired() { // If a stretch cluster, a zone can override the template from the default. var zones []cephv1.MonZoneSpec diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 7e7cb91f9..64f3685d2 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -63,8 +63,8 @@ func (c *Cluster) getLabels(monConfig *monConfig, canary, includeNewLabels bool) return labels } -func (c *Cluster) stretchFailureDomainName() string { - label := StretchFailureDomainLabel(c.spec) +func (c *Cluster) getFailureDomainName() string { + label := GetFailureDomainLabel(c.spec) index := strings.Index(label, "/") if index == -1 { return label @@ -72,10 +72,14 @@ func (c *Cluster) stretchFailureDomainName() string { return label[index+1:] } -func StretchFailureDomainLabel(spec cephv1.ClusterSpec) string { +func GetFailureDomainLabel(spec cephv1.ClusterSpec) string { if spec.Mon.StretchCluster.FailureDomainLabel != "" { return spec.Mon.StretchCluster.FailureDomainLabel } + if spec.Mon.FailureDomainLabel != "" { + logger.Infof("the failure domain for mon zone is set to %s", spec.Mon.FailureDomainLabel) + return spec.Mon.FailureDomainLabel + } // The default topology label is for a zone return corev1.LabelZoneFailureDomainStable } @@ -217,8 +221,8 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, er } } - if c.spec.IsStretchCluster() { - nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", StretchFailureDomainLabel(c.spec), monConfig.Zone)) + if c.spec.ZonesRequired() { + nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", GetFailureDomainLabel(c.spec), monConfig.Zone)) if err != nil { return nil, errors.Wrapf(err, "failed to generate mon %q node affinity", monConfig.DaemonName) } @@ -322,7 +326,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container } if monConfig.Zone != "" { - desiredLocation := fmt.Sprintf("%s=%s", c.stretchFailureDomainName(), monConfig.Zone) + desiredLocation := fmt.Sprintf("%s=%s", c.getFailureDomainName(), monConfig.Zone) container.Args = append(container.Args, []string{"--set-crush-location", desiredLocation}...) if monConfig.Zone == c.getArbiterZone() { // remember the arbiter mon to be set later in the reconcile after the OSDs are configured