mon: add failure domain label for zonal topology

Signed-off-by: Deepika Upadhyay <deepika@koor.tech>
(cherry picked from commit 764f6597c0)

Documentation/CRDs/specification.md

@@ -7490,6 +7490,17 @@ bool
 </tr>
 <tr>
 <td>
+<code>failureDomainLabel</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+</td>
+</tr>
+<tr>
+<td>
 <code>zones</code><br/>
 <em>
 <a href="#ceph.rook.io/v1.MonZoneSpec">

pkg/apis/ceph.rook.io/v1/cluster.go

@@ -50,12 +50,21 @@ func (c *ClusterSpec) IsStretchCluster() bool {
 	return c.Mon.StretchCluster != nil && len(c.Mon.StretchCluster.Zones) > 0
 }
 
+func (c *ClusterSpec) ZonesRequired() bool {
+	return c.IsStretchCluster() || len(c.Mon.Zones) > 0
+}
+
 func (c *CephCluster) ValidateCreate() (admission.Warnings, error) {
+	if c.Spec.ZonesRequired() {
+		if len(c.Spec.Mon.Zones) < c.Spec.Mon.Count {
+			return nil, errors.New("Not enough zones available for mons, please specify more zones")
+		}
+	}
 	logger.Infof("validate create cephcluster %q", c.ObjectMeta.Name)
 	//If external mode enabled, then check if other fields are empty
 	if c.Spec.External.Enable {
 		if !reflect.DeepEqual(c.Spec.Mon, MonSpec{}) || c.Spec.Dashboard != (DashboardSpec{}) || !reflect.DeepEqual(c.Spec.Monitoring, (MonitoringSpec{})) || c.Spec.DisruptionManagement != (DisruptionManagementSpec{}) || len(c.Spec.Mgr.Modules) > 0 || len(c.Spec.Network.Provider) > 0 || len(c.Spec.Network.Selectors) > 0 {
-			return errors.New("invalid create : external mode enabled cannot have mon,dashboard,monitoring,network,disruptionManagement,storage fields in CR")
+			return nil, errors.New("invalid create : external mode enabled cannot have mon,dashboard,monitoring,network,disruptionManagement,storage fields in CR")
 		}
 	}
 	return nil, nil

pkg/apis/ceph.rook.io/v1/types.go

@@ -530,6 +530,8 @@ type MonSpec struct {
 	// AllowMultiplePerNode determines if we can run multiple monitors on the same node (not recommended)
 	// +optional
 	AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"`
+	// +optional
+	FailureDomainLabel string `json:"failureDomainLabel,omitempty"`
 	// Zones are specified when we want to provide zonal awareness to mons
 	// +optional
 	Zones []MonZoneSpec `json:"zones,omitempty"`

pkg/operator/ceph/cluster/mgr/spec.go

@@ -76,10 +76,10 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
 		matchLabels := controller.AppLabels(AppName, c.clusterInfo.Namespace)
 		podSpec.Spec.Volumes = append(podSpec.Spec.Volumes, mon.CephSecretVolume())
 
-		// Stretch the mgrs across hosts by default, or across a bigger failure domain for stretch clusters
+		// Stretch the mgrs across hosts by default, or across a bigger failure domain for when zones are required like in case of stretched cluster
 		topologyKey := v1.LabelHostname
-		if c.spec.IsStretchCluster() {
-			topologyKey = mon.StretchFailureDomainLabel(c.spec)
+		if c.spec.ZonesRequired() {
+			topologyKey = mon.GetFailureDomainLabel(c.spec)
 		}
 		k8sutil.SetNodeAntiAffinityForPod(&podSpec.Spec, !c.spec.Mgr.AllowMultiplePerNode, topologyKey, matchLabels, nil)
 	}

pkg/operator/ceph/cluster/mon/mon.go

@@ -314,7 +314,7 @@ func (c *Cluster) configureStretchCluster(mons []*monConfig) error {
 	}
 
 	// Create the default crush rule for stretch clusters, that by default will also apply to all pools
-	if err := cephclient.CreateDefaultStretchCrushRule(c.context, c.ClusterInfo, &c.spec, c.stretchFailureDomainName()); err != nil {
+	if err := cephclient.CreateDefaultStretchCrushRule(c.context, c.ClusterInfo, &c.spec, c.getFailureDomainName()); err != nil {
 		return errors.Wrap(err, "failed to create default stretch rule")
 	}
 
@@ -367,7 +367,7 @@ func (c *Cluster) ConfigureArbiter() error {
 	// Wait for the CRUSH map to have at least two zones
 	// The timeout is relatively short since the operator will requeue the reconcile
 	// and try again at a higher level if not yet found
-	failureDomain := c.stretchFailureDomainName()
+	failureDomain := c.getFailureDomainName()
 	logger.Infof("enabling stretch mode... waiting for two failure domains of type %q to be found in the CRUSH map after OSD initialization", failureDomain)
 	pollInterval := 5 * time.Second
 	totalWaitTime := 2 * time.Minute
@@ -387,7 +387,7 @@ func (c *Cluster) ConfigureArbiter() error {
 }
 
 func (c *Cluster) readyToConfigureArbiter(checkOSDPods bool) (bool, error) {
-	failureDomain := c.stretchFailureDomainName()
+	failureDomain := c.getFailureDomainName()
 
 	if checkOSDPods {
 		// Wait for the OSD pods to be running
@@ -596,7 +596,7 @@ func (c *Cluster) newMonConfig(monID int, zone string) *monConfig {
 }
 
 func (c *Cluster) findAvailableZone(mons []*monConfig) (string, error) {
-	if !c.spec.IsStretchCluster() && !(len(c.spec.Mon.Zones) > 0) {
+	if !c.spec.ZonesRequired() {
 		return "", nil
 	}
 
@@ -926,7 +926,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
 			} else {
 				logger.Infof("mon %q placement using native scheduler", mon.DaemonName)
 			}
-			if (len(c.spec.Mon.Zones) > 0) || c.spec.IsStretchCluster() {
+			if c.spec.ZonesRequired() {
 				if schedule == nil {
 					schedule = &controller.MonScheduleInfo{}
 				}
@@ -952,7 +952,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
 
 func (c *Cluster) monVolumeClaimTemplate(mon *monConfig) *v1.PersistentVolumeClaim {
 
-	if c.spec.IsStretchCluster() || len(c.spec.Mon.Zones) > 0 {
+	if c.spec.ZonesRequired() {
 		// If a stretch cluster, a zone can override the template from the default.
 
 		var zones []cephv1.MonZoneSpec

pkg/operator/ceph/cluster/mon/spec.go

@@ -63,8 +63,8 @@ func (c *Cluster) getLabels(monConfig *monConfig, canary, includeNewLabels bool)
 	return labels
 }
 
-func (c *Cluster) stretchFailureDomainName() string {
-	label := StretchFailureDomainLabel(c.spec)
+func (c *Cluster) getFailureDomainName() string {
+	label := GetFailureDomainLabel(c.spec)
 	index := strings.Index(label, "/")
 	if index == -1 {
 		return label
@@ -72,10 +72,14 @@ func (c *Cluster) stretchFailureDomainName() string {
 	return label[index+1:]
 }
 
-func StretchFailureDomainLabel(spec cephv1.ClusterSpec) string {
+func GetFailureDomainLabel(spec cephv1.ClusterSpec) string {
 	if spec.Mon.StretchCluster.FailureDomainLabel != "" {
 		return spec.Mon.StretchCluster.FailureDomainLabel
 	}
+	if spec.Mon.FailureDomainLabel != "" {
+		logger.Infof("the failure domain for mon zone is set to %s", spec.Mon.FailureDomainLabel)
+		return spec.Mon.FailureDomainLabel
+	}
 	// The default topology label is for a zone
 	return corev1.LabelZoneFailureDomainStable
 }
@@ -217,8 +221,8 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, er
 		}
 	}
 
-	if c.spec.IsStretchCluster() {
-		nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", StretchFailureDomainLabel(c.spec), monConfig.Zone))
+	if c.spec.ZonesRequired() {
+		nodeAffinity, err := k8sutil.GenerateNodeAffinity(fmt.Sprintf("%s=%s", GetFailureDomainLabel(c.spec), monConfig.Zone))
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to generate mon %q node affinity", monConfig.DaemonName)
 		}
@@ -322,7 +326,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
 	}
 
 	if monConfig.Zone != "" {
-		desiredLocation := fmt.Sprintf("%s=%s", c.stretchFailureDomainName(), monConfig.Zone)
+		desiredLocation := fmt.Sprintf("%s=%s", c.getFailureDomainName(), monConfig.Zone)
 		container.Args = append(container.Args, []string{"--set-crush-location", desiredLocation}...)
 		if monConfig.Zone == c.getArbiterZone() {
 			// remember the arbiter mon to be set later in the reconcile after the OSDs are configured