ceph: allow passing 'osd-crush-initial-weight'

Ceph support the option '--osd-crush-initial-weight' upon OSD start,
which sets an explicit weight (in TiB units) to specific OSD. Allow
passing this option all the way from the user (similar to
'DeviceClass'), for the special case where end users wants it cluster
to have non-even balance over specific OSDs (e.g., one of the OSDs is
placed over a partition alongside OS-partition).

ROOK issue: https://github.com/rook/rook/issues/7448

Signed-off-by: Shachar Sharon <ssharon@redhat.com>

cluster/charts/rook-ceph/templates/resources.yaml

@@ -3160,6 +3160,10 @@ spec:
                           crushDeviceClass:
                             description: CrushDeviceClass represents the crush device class for an OSD
                             type: string
+                          crushInitialWeight:
+                            description: CrushInitialWeight represents initial OSD weight in TiB units
+                            pattern: ^([0-9]*[.])?[0-9]$
+                            type: string
                           encrypted:
                             description: Whether to encrypt the deviceSet
                             type: boolean

cluster/examples/kubernetes/ceph/crds.yaml

@@ -3162,6 +3162,10 @@ spec:
                           crushDeviceClass:
                             description: CrushDeviceClass represents the crush device class for an OSD
                             type: string
+                          crushInitialWeight:
+                            description: CrushInitialWeight represents initial OSD weight in TiB units
+                            pattern: ^([0-9]*[.])?[0-9]$
+                            type: string
                           encrypted:
                             description: Whether to encrypt the deviceSet
                             type: boolean

cmd/rook/ceph/osd.go

@@ -118,6 +118,7 @@ func addOSDConfigFlags(command *cobra.Command) {
 	command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device")
 	command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt")
 	command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node")
+	command.Flags().StringVar(&cfg.storeConfig.InitialWeight, "osd-crush-initial-weight", "", "The initial weight of OSD in TiB units")
 }
 
 func init() {
@@ -303,6 +304,7 @@ func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) {
 		d.OSDsPerDevice = cd.StoreConfig.OSDsPerDevice
 		d.DatabaseSizeMB = cd.StoreConfig.DatabaseSizeMB
 		d.DeviceClass = cd.StoreConfig.DeviceClass
+		d.InitialWeight = cd.StoreConfig.InitialWeight
 		d.MetadataDevice = cd.StoreConfig.MetadataDevice
 
 		if d.OSDsPerDevice < 1 {

pkg/apis/rook.io/v1/types.go

@@ -217,6 +217,10 @@ type VolumeSource struct {
 	// CrushDeviceClass represents the crush device class for an OSD
 	// +optional
 	CrushDeviceClass string `json:"crushDeviceClass,omitempty"`
+	// CrushInitialWeight represents initial OSD weight in TiB units
+	// +kubebuilder:validation:Pattern=`^([0-9]*[.])?[0-9]$`
+	// +optional
+	CrushInitialWeight string `json:"crushInitialWeight,omitempty"`
 	// Size represents the size requested for the PVC
 	Size string `json:"size"`
 	// Resources requests/limits for the devices

pkg/daemon/ceph/osd/device.go

@@ -42,6 +42,7 @@ type DesiredDevice struct {
 	MetadataDevice     string
 	DatabaseSizeMB     int
 	DeviceClass        string
+	InitialWeight      string
 	IsFilter           bool
 	IsDevicePathFilter bool
 }

pkg/operator/ceph/cluster/osd/config/config.go

@@ -29,6 +29,7 @@ const (
 	EncryptedDeviceKey = "encryptedDevice"
 	MetadataDeviceKey  = "metadataDevice"
 	DeviceClassKey     = "deviceClass"
+	InitialWeightKey   = "initialWeight"
 )
 
 // StoreConfig represents the configuration of an OSD on a device.
@@ -39,6 +40,7 @@ type StoreConfig struct {
 	EncryptedDevice bool   `json:"encryptedDevice,omitempty"`
 	MetadataDevice  string `json:"metadataDevice,omitempty"`
 	DeviceClass     string `json:"deviceClass,omitempty"`
+	InitialWeight   string `json:"initialWeight,omitempty"`
 }
 
 // NewStoreConfig returns a StoreConfig with proper defaults set.
@@ -68,6 +70,8 @@ func ToStoreConfig(config map[string]string) StoreConfig {
 			storeConfig.MetadataDevice = v
 		case DeviceClassKey:
 			storeConfig.DeviceClass = v
+		case InitialWeightKey:
+			storeConfig.InitialWeight = v
 		}
 	}
 

pkg/operator/ceph/cluster/osd/deviceSet.go

@@ -96,6 +96,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic
 
 	var dataSize string
 	var crushDeviceClass string
+	var crushInitialWeight string
 	typesFound := util.NewSet()
 	for _, pvcTemplate := range deviceSet.VolumeClaimTemplates {
 		if pvcTemplate.Name == "" {
@@ -126,6 +127,8 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic
 			dataSize = pvcSize.String()
 			crushDeviceClass = pvcTemplate.Annotations["crushDeviceClass"]
 		}
+		crushInitialWeight = pvcTemplate.Annotations["crushInitialWeight"]
+
 		pvcSources[pvcType] = v1.PersistentVolumeClaimVolumeSource{
 			ClaimName: pvc.GetName(),
 			ReadOnly:  false,
@@ -145,6 +148,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic
 		TuneFastDeviceClass: deviceSet.TuneFastDeviceClass,
 		SchedulerName:       deviceSet.SchedulerName,
 		CrushDeviceClass:    crushDeviceClass,
+		CrushInitialWeight:  crushInitialWeight,
 		Encrypted:           deviceSet.Encrypted,
 	}
 }

pkg/operator/ceph/cluster/osd/deviceset_test.go

@@ -269,3 +269,37 @@ func TestUpdatePVCSize(t *testing.T) {
 	assert.True(t, ok)
 	assert.Equal(t, "6Gi", result.String())
 }
+
+func TestPrepareDeviceSetsWithCrushParams(t *testing.T) {
+	ctx := context.TODO()
+	clientset := testexec.New(t, 1)
+	context := &clusterd.Context{
+		Clientset: clientset,
+	}
+	deviceSet := rookv1.StorageClassDeviceSet{
+		Name:                 "datawithcrushparams1",
+		Count:                1,
+		VolumeClaimTemplates: []corev1.PersistentVolumeClaim{testVolumeClaim("testwithcrushparams1")},
+		SchedulerName:        "custom-scheduler",
+	}
+	deviceSet.VolumeClaimTemplates[0].Annotations = map[string]string{"crushDeviceClass": "ssd", "crushInitialWeight": "0.75"}
+
+	spec := cephv1.ClusterSpec{
+		Storage: rookv1.StorageScopeSpec{StorageClassDeviceSets: []rookv1.StorageClassDeviceSet{deviceSet}},
+	}
+	cluster := &Cluster{
+		context:     context,
+		clusterInfo: client.AdminClusterInfo("testns"),
+		spec:        spec,
+	}
+
+	config := newProvisionErrors()
+	volumeSources := cluster.prepareStorageClassDeviceSets(config)
+	assert.Equal(t, 1, len(volumeSources))
+	assert.Equal(t, volumeSources[0].CrushDeviceClass, "ssd")
+	assert.Equal(t, volumeSources[0].CrushInitialWeight, "0.75")
+
+	pvcs, err := clientset.CoreV1().PersistentVolumeClaims(cluster.clusterInfo.Namespace).List(ctx, metav1.ListOptions{})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(pvcs.Items))
+}

pkg/operator/ceph/cluster/osd/envs.go

@@ -45,6 +45,7 @@ const (
 	cvModeVarName                       = "ROOK_CV_MODE"
 	lvBackedPVVarName                   = "ROOK_LV_BACKED_PV"
 	CrushDeviceClassVarName             = "ROOK_OSD_CRUSH_DEVICE_CLASS"
+	CrushInitialWeightVarName           = "ROOK_OSD_CRUSH_INITIAL_WEIGHT"
 	CrushRootVarName                    = "ROOK_CRUSHMAP_ROOT"
 	tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES"
 )
@@ -159,6 +160,10 @@ func crushDeviceClassEnvVar(crushDeviceClass string) v1.EnvVar {
 	return v1.EnvVar{Name: CrushDeviceClassVarName, Value: crushDeviceClass}
 }
 
+func crushInitialWeightEnvVar(crushInitialWeight string) v1.EnvVar {
+	return v1.EnvVar{Name: CrushInitialWeightVarName, Value: crushInitialWeight}
+}
+
 func encryptedDeviceEnvVar(encryptedDevice bool) v1.EnvVar {
 	return v1.EnvVar{Name: EncryptedDeviceEnvVarName, Value: strconv.FormatBool(encryptedDevice)}
 }

pkg/operator/ceph/cluster/osd/osd.go

@@ -363,6 +363,8 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) {
 				encrypted:           volumeSource.Encrypted,
 				deviceSetName:       volumeSource.Name,
 			}
+			osdProps.storeConfig.InitialWeight = volumeSource.CrushInitialWeight
+
 			// If OSD isn't portable, we're getting the host name either from the osd deployment that was already initialized
 			// or from the osd prepare job from initial creation.
 			if !volumeSource.Portable {

pkg/operator/ceph/cluster/osd/provision_spec.go

@@ -229,6 +229,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
 	}
 	envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()})
 	envVars = append(envVars, crushDeviceClassEnvVar(osdProps.storeConfig.DeviceClass))
+	envVars = append(envVars, crushInitialWeightEnvVar(osdProps.storeConfig.InitialWeight))
 
 	if osdProps.metadataDevice != "" {
 		envVars = append(envVars, metadataDeviceEnvVar(osdProps.metadataDevice))

pkg/operator/ceph/cluster/osd/spec.go

@@ -404,6 +404,11 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 		}
 	}
 
+	// Ceph expects initial weight as float value in tera-bytes units
+	if osdProps.storeConfig.InitialWeight != "" {
+		args = append(args, fmt.Sprintf("--osd-crush-initial-weight=%s", osdProps.storeConfig.InitialWeight))
+	}
+
 	// If the OSD runs on PVC
 	if osdProps.onPVC() {
 		// add the PVC size to the pod spec so that if the size changes the OSD will be restarted and pick up the change