Merge pull request #7807 from synarete/osd-primary-affinity

ceph: allow setting primary-affinity to osd

Documentation/ceph-cluster-crd.md

@@ -464,6 +464,8 @@ The following storage selection settings are specific to Ceph and do not apply t
 * `databaseSizeMB`:  The size in MB of a bluestore database. Include quotes around the size.
 * `walSizeMB`:  The size in MB of a bluestore write ahead log (WAL). Include quotes around the size.
 * `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme`  based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](ceph-pool-crd.md#spec).
+* `initialWeight`: The initial OSD weight in TiB units. By default, this value is derived from OSD's capacity.
+* `primaryAffinity`: The [primary-affinity](https://docs.ceph.com/en/latest/rados/operations/crush-map/#primary-affinity) value of an OSD, within range `[0, 1]` (default: `1`).
 * `osdsPerDevice`**: The number of OSDs to create on each device. High performance devices such as NVMe can handle running multiple OSDs. If desired, this can be overridden for each node and each device.
 * `encryptedDevice`**: Encrypt OSD volumes using dmcrypt ("true" or "false"). By default this option is disabled. See [encryption](http://docs.ceph.com/docs/nautilus/ceph-volume/lvm/encryption/) for more information on encryption in Ceph.
 * `crushRoot`: The value of the `root` CRUSH map label. The default is `default`. Generally, you should not need to change this. However, if any of your topology labels may have the value `default`, you need to change `crushRoot` to avoid conflicts, since CRUSH map values need to be unique.

pkg/daemon/ceph/client/osd.go

@@ -355,3 +355,15 @@ func OSDOkToStop(context *clusterd.Context, clusterInfo *ClusterInfo, osdID, max
 
 	return stats.OSDs, nil
 }
+
+// SetPrimaryAffinity assigns primary-affinity (within range [0.0, 1.0]) to a specific OSD.
+func SetPrimaryAffinity(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int, affinity string) error {
+	logger.Infof("setting osd.%d with primary-affinity %q", osdID, affinity)
+	args := []string{"osd", "primary-affinity", fmt.Sprintf("osd.%d", osdID), affinity}
+	_, err := NewCephCommand(context, clusterInfo, args).Run()
+	if err != nil {
+		return errors.Wrapf(err, "failed to set osd.%d with primary-affinity %q", osdID, affinity)
+	}
+	logger.Infof("successfully applied osd.%d primary-affinity %q", osdID, affinity)
+	return nil
+}

pkg/operator/ceph/cluster/osd/config/config.go

@@ -30,6 +30,7 @@ const (
 	MetadataDeviceKey  = "metadataDevice"
 	DeviceClassKey     = "deviceClass"
 	InitialWeightKey   = "initialWeight"
+	PrimaryAffinityKey = "primaryAffinity"
 )
 
 // StoreConfig represents the configuration of an OSD on a device.
@@ -41,6 +42,7 @@ type StoreConfig struct {
 	MetadataDevice  string `json:"metadataDevice,omitempty"`
 	DeviceClass     string `json:"deviceClass,omitempty"`
 	InitialWeight   string `json:"initialWeight,omitempty"`
+	PrimaryAffinity string `json:"primaryAffinity,omitempty"`
 }
 
 // NewStoreConfig returns a StoreConfig with proper defaults set.
@@ -72,6 +74,8 @@ func ToStoreConfig(config map[string]string) StoreConfig {
 			storeConfig.DeviceClass = v
 		case InitialWeightKey:
 			storeConfig.InitialWeight = v
+		case PrimaryAffinityKey:
+			storeConfig.PrimaryAffinity = v
 		}
 	}
 

pkg/operator/ceph/cluster/osd/deviceSet.go

@@ -43,6 +43,8 @@ type deviceSet struct {
 	CrushDeviceClass string
 	// CrushInitialWeight represents initial OSD weight in TiB units
 	CrushInitialWeight string
+	// CrushPrimaryAffinity represents initial OSD primary-affinity within range [0, 1]
+	CrushPrimaryAffinity string
 	// Size represents the size requested for the PVC
 	Size string
 	// Resources requests/limits for the devices
@@ -128,6 +130,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(newDeviceSet rookv1.StorageClassDe
 	var dataSize string
 	var crushDeviceClass string
 	var crushInitialWeight string
+	var crushPrimaryAffinity string
 	typesFound := util.NewSet()
 	for _, pvcTemplate := range newDeviceSet.VolumeClaimTemplates {
 		if pvcTemplate.Name == "" {
@@ -159,6 +162,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(newDeviceSet rookv1.StorageClassDe
 			crushDeviceClass = pvcTemplate.Annotations["crushDeviceClass"]
 		}
 		crushInitialWeight = pvcTemplate.Annotations["crushInitialWeight"]
+		crushPrimaryAffinity = pvcTemplate.Annotations["crushPrimaryAffinity"]
 
 		pvcSources[pvcType] = v1.PersistentVolumeClaimVolumeSource{
 			ClaimName: pvc.GetName(),
@@ -167,20 +171,21 @@ func (c *Cluster) createDeviceSetPVCsForIndex(newDeviceSet rookv1.StorageClassDe
 	}
 
 	return deviceSet{
-		Name:                newDeviceSet.Name,
-		Resources:           newDeviceSet.Resources,
-		Placement:           newDeviceSet.Placement,
-		PreparePlacement:    newDeviceSet.PreparePlacement,
-		Config:              newDeviceSet.Config,
-		Size:                dataSize,
-		PVCSources:          pvcSources,
-		Portable:            newDeviceSet.Portable,
-		TuneSlowDeviceClass: newDeviceSet.TuneSlowDeviceClass,
-		TuneFastDeviceClass: newDeviceSet.TuneFastDeviceClass,
-		SchedulerName:       newDeviceSet.SchedulerName,
-		CrushDeviceClass:    crushDeviceClass,
-		CrushInitialWeight:  crushInitialWeight,
-		Encrypted:           newDeviceSet.Encrypted,
+		Name:                 newDeviceSet.Name,
+		Resources:            newDeviceSet.Resources,
+		Placement:            newDeviceSet.Placement,
+		PreparePlacement:     newDeviceSet.PreparePlacement,
+		Config:               newDeviceSet.Config,
+		Size:                 dataSize,
+		PVCSources:           pvcSources,
+		Portable:             newDeviceSet.Portable,
+		TuneSlowDeviceClass:  newDeviceSet.TuneSlowDeviceClass,
+		TuneFastDeviceClass:  newDeviceSet.TuneFastDeviceClass,
+		SchedulerName:        newDeviceSet.SchedulerName,
+		CrushDeviceClass:     crushDeviceClass,
+		CrushInitialWeight:   crushInitialWeight,
+		CrushPrimaryAffinity: crushPrimaryAffinity,
+		Encrypted:            newDeviceSet.Encrypted,
 	}
 }
 

pkg/operator/ceph/cluster/osd/deviceset_test.go

@@ -245,7 +245,11 @@ func TestPrepareDeviceSetsWithCrushParams(t *testing.T) {
 		VolumeClaimTemplates: []corev1.PersistentVolumeClaim{testVolumeClaim("testwithcrushparams1")},
 		SchedulerName:        "custom-scheduler",
 	}
-	deviceSet.VolumeClaimTemplates[0].Annotations = map[string]string{"crushDeviceClass": "ssd", "crushInitialWeight": "0.75"}
+	deviceSet.VolumeClaimTemplates[0].Annotations = map[string]string{
+		"crushDeviceClass":     "ssd",
+		"crushInitialWeight":   "0.75",
+		"crushPrimaryAffinity": "0.666",
+	}
 
 	spec := cephv1.ClusterSpec{
 		Storage: rookv1.StorageScopeSpec{StorageClassDeviceSets: []rookv1.StorageClassDeviceSet{deviceSet}},
@@ -261,6 +265,7 @@ func TestPrepareDeviceSetsWithCrushParams(t *testing.T) {
 	assert.Equal(t, 1, len(cluster.deviceSets))
 	assert.Equal(t, cluster.deviceSets[0].CrushDeviceClass, "ssd")
 	assert.Equal(t, cluster.deviceSets[0].CrushInitialWeight, "0.75")
+	assert.Equal(t, cluster.deviceSets[0].CrushPrimaryAffinity, "0.666")
 
 	pvcs, err := clientset.CoreV1().PersistentVolumeClaims(cluster.clusterInfo.Namespace).List(ctx, metav1.ListOptions{})
 	assert.NoError(t, err)

pkg/operator/ceph/cluster/osd/osd.go

@@ -267,6 +267,11 @@ func deploymentOnNode(c *Cluster, osd OSDInfo, nodeName string, config *provisio
 		return nil, errors.Wrapf(err, "failed to generate deployment for %s", osdLongName)
 	}
 
+	err = setOSDProperties(c, osdProps, osd)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to prepare deployment for %s", osdLongName)
+	}
+
 	return d, nil
 }
 
@@ -283,9 +288,24 @@ func deploymentOnPVC(c *Cluster, osd OSDInfo, pvcName string, config *provisionC
 		return nil, errors.Wrapf(err, "failed to generate deployment for %s", osdLongName)
 	}
 
+	err = setOSDProperties(c, osdProps, osd)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to prepare deployment for %s", osdLongName)
+	}
+
 	return d, nil
 }
 
+// setOSDProperties is used to configure an OSD with parameters which can not be set via explicit
+// command-line arguments.
+func setOSDProperties(c *Cluster, osdProps osdProperties, osd OSDInfo) error {
+	// OSD's 'primary-affinity' has to be configured via command which goes through mons
+	if osdProps.storeConfig.PrimaryAffinity != "" {
+		return cephclient.SetPrimaryAffinity(c.context, c.clusterInfo, osd.ID, osdProps.storeConfig.PrimaryAffinity)
+	}
+	return nil
+}
+
 func (c *Cluster) resolveNode(nodeName string) *rookv1.Node {
 	// fully resolve the storage config and resources for this node
 	rookNode := c.ValidStorage.ResolveNode(nodeName)
@@ -361,6 +381,7 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) {
 				deviceSetName:       deviceSet.Name,
 			}
 			osdProps.storeConfig.InitialWeight = deviceSet.CrushInitialWeight
+			osdProps.storeConfig.PrimaryAffinity = deviceSet.CrushPrimaryAffinity
 
 			// If OSD isn't portable, we're getting the host name either from the osd deployment that was already initialized
 			// or from the osd prepare job from initial creation.