diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 277afe08e..5c736f7c2 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -3160,6 +3160,10 @@ spec: crushDeviceClass: description: CrushDeviceClass represents the crush device class for an OSD type: string + crushInitialWeight: + description: CrushInitialWeight represents initial OSD weight in TiB units + pattern: ^([0-9]*[.])?[0-9]$ + type: string encrypted: description: Whether to encrypt the deviceSet type: boolean diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 2b779d25f..69ce3afe7 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -3162,6 +3162,10 @@ spec: crushDeviceClass: description: CrushDeviceClass represents the crush device class for an OSD type: string + crushInitialWeight: + description: CrushInitialWeight represents initial OSD weight in TiB units + pattern: ^([0-9]*[.])?[0-9]$ + type: string encrypted: description: Whether to encrypt the deviceSet type: boolean diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 203cdedda..38071f4d9 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -118,6 +118,7 @@ func addOSDConfigFlags(command *cobra.Command) { command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device") command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt") command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node") + command.Flags().StringVar(&cfg.storeConfig.InitialWeight, "osd-crush-initial-weight", "", "The initial weight of OSD in TiB units") } func init() { @@ -303,6 +304,7 @@ func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) { d.OSDsPerDevice = cd.StoreConfig.OSDsPerDevice d.DatabaseSizeMB = cd.StoreConfig.DatabaseSizeMB d.DeviceClass = cd.StoreConfig.DeviceClass + d.InitialWeight = cd.StoreConfig.InitialWeight d.MetadataDevice = cd.StoreConfig.MetadataDevice if d.OSDsPerDevice < 1 { diff --git a/pkg/apis/rook.io/v1/types.go b/pkg/apis/rook.io/v1/types.go index 054b2d601..40b47df85 100644 --- a/pkg/apis/rook.io/v1/types.go +++ b/pkg/apis/rook.io/v1/types.go @@ -217,6 +217,10 @@ type VolumeSource struct { // CrushDeviceClass represents the crush device class for an OSD // +optional CrushDeviceClass string `json:"crushDeviceClass,omitempty"` + // CrushInitialWeight represents initial OSD weight in TiB units + // +kubebuilder:validation:Pattern=`^([0-9]*[.])?[0-9]$` + // +optional + CrushInitialWeight string `json:"crushInitialWeight,omitempty"` // Size represents the size requested for the PVC Size string `json:"size"` // Resources requests/limits for the devices diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index d6bf8a7eb..29e937487 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -42,6 +42,7 @@ type DesiredDevice struct { MetadataDevice string DatabaseSizeMB int DeviceClass string + InitialWeight string IsFilter bool IsDevicePathFilter bool } diff --git a/pkg/operator/ceph/cluster/osd/config/config.go b/pkg/operator/ceph/cluster/osd/config/config.go index c706f4a1d..d0daa4dc1 100644 --- a/pkg/operator/ceph/cluster/osd/config/config.go +++ b/pkg/operator/ceph/cluster/osd/config/config.go @@ -29,6 +29,7 @@ const ( EncryptedDeviceKey = "encryptedDevice" MetadataDeviceKey = "metadataDevice" DeviceClassKey = "deviceClass" + InitialWeightKey = "initialWeight" ) // StoreConfig represents the configuration of an OSD on a device. @@ -39,6 +40,7 @@ type StoreConfig struct { EncryptedDevice bool `json:"encryptedDevice,omitempty"` MetadataDevice string `json:"metadataDevice,omitempty"` DeviceClass string `json:"deviceClass,omitempty"` + InitialWeight string `json:"initialWeight,omitempty"` } // NewStoreConfig returns a StoreConfig with proper defaults set. @@ -68,6 +70,8 @@ func ToStoreConfig(config map[string]string) StoreConfig { storeConfig.MetadataDevice = v case DeviceClassKey: storeConfig.DeviceClass = v + case InitialWeightKey: + storeConfig.InitialWeight = v } } diff --git a/pkg/operator/ceph/cluster/osd/deviceSet.go b/pkg/operator/ceph/cluster/osd/deviceSet.go index f73d672f0..ca4c2b795 100644 --- a/pkg/operator/ceph/cluster/osd/deviceSet.go +++ b/pkg/operator/ceph/cluster/osd/deviceSet.go @@ -96,6 +96,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic var dataSize string var crushDeviceClass string + var crushInitialWeight string typesFound := util.NewSet() for _, pvcTemplate := range deviceSet.VolumeClaimTemplates { if pvcTemplate.Name == "" { @@ -126,6 +127,8 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic dataSize = pvcSize.String() crushDeviceClass = pvcTemplate.Annotations["crushDeviceClass"] } + crushInitialWeight = pvcTemplate.Annotations["crushInitialWeight"] + pvcSources[pvcType] = v1.PersistentVolumeClaimVolumeSource{ ClaimName: pvc.GetName(), ReadOnly: false, @@ -145,6 +148,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic TuneFastDeviceClass: deviceSet.TuneFastDeviceClass, SchedulerName: deviceSet.SchedulerName, CrushDeviceClass: crushDeviceClass, + CrushInitialWeight: crushInitialWeight, Encrypted: deviceSet.Encrypted, } } diff --git a/pkg/operator/ceph/cluster/osd/deviceset_test.go b/pkg/operator/ceph/cluster/osd/deviceset_test.go index 0d06c00bc..8ae0582f9 100644 --- a/pkg/operator/ceph/cluster/osd/deviceset_test.go +++ b/pkg/operator/ceph/cluster/osd/deviceset_test.go @@ -269,3 +269,37 @@ func TestUpdatePVCSize(t *testing.T) { assert.True(t, ok) assert.Equal(t, "6Gi", result.String()) } + +func TestPrepareDeviceSetsWithCrushParams(t *testing.T) { + ctx := context.TODO() + clientset := testexec.New(t, 1) + context := &clusterd.Context{ + Clientset: clientset, + } + deviceSet := rookv1.StorageClassDeviceSet{ + Name: "datawithcrushparams1", + Count: 1, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{testVolumeClaim("testwithcrushparams1")}, + SchedulerName: "custom-scheduler", + } + deviceSet.VolumeClaimTemplates[0].Annotations = map[string]string{"crushDeviceClass": "ssd", "crushInitialWeight": "0.75"} + + spec := cephv1.ClusterSpec{ + Storage: rookv1.StorageScopeSpec{StorageClassDeviceSets: []rookv1.StorageClassDeviceSet{deviceSet}}, + } + cluster := &Cluster{ + context: context, + clusterInfo: client.AdminClusterInfo("testns"), + spec: spec, + } + + config := newProvisionErrors() + volumeSources := cluster.prepareStorageClassDeviceSets(config) + assert.Equal(t, 1, len(volumeSources)) + assert.Equal(t, volumeSources[0].CrushDeviceClass, "ssd") + assert.Equal(t, volumeSources[0].CrushInitialWeight, "0.75") + + pvcs, err := clientset.CoreV1().PersistentVolumeClaims(cluster.clusterInfo.Namespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + assert.Equal(t, 1, len(pvcs.Items)) +} diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index 677fcea76..b8d9b04b9 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -45,6 +45,7 @@ const ( cvModeVarName = "ROOK_CV_MODE" lvBackedPVVarName = "ROOK_LV_BACKED_PV" CrushDeviceClassVarName = "ROOK_OSD_CRUSH_DEVICE_CLASS" + CrushInitialWeightVarName = "ROOK_OSD_CRUSH_INITIAL_WEIGHT" CrushRootVarName = "ROOK_CRUSHMAP_ROOT" tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES" ) @@ -159,6 +160,10 @@ func crushDeviceClassEnvVar(crushDeviceClass string) v1.EnvVar { return v1.EnvVar{Name: CrushDeviceClassVarName, Value: crushDeviceClass} } +func crushInitialWeightEnvVar(crushInitialWeight string) v1.EnvVar { + return v1.EnvVar{Name: CrushInitialWeightVarName, Value: crushInitialWeight} +} + func encryptedDeviceEnvVar(encryptedDevice bool) v1.EnvVar { return v1.EnvVar{Name: EncryptedDeviceEnvVarName, Value: strconv.FormatBool(encryptedDevice)} } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index ac3d3ac23..fa6d378f5 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -363,6 +363,8 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) { encrypted: volumeSource.Encrypted, deviceSetName: volumeSource.Name, } + osdProps.storeConfig.InitialWeight = volumeSource.CrushInitialWeight + // If OSD isn't portable, we're getting the host name either from the osd deployment that was already initialized // or from the osd prepare job from initial creation. if !volumeSource.Portable { diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 2035f1ab3..e1f0f50ae 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -229,6 +229,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun } envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()}) envVars = append(envVars, crushDeviceClassEnvVar(osdProps.storeConfig.DeviceClass)) + envVars = append(envVars, crushInitialWeightEnvVar(osdProps.storeConfig.InitialWeight)) if osdProps.metadataDevice != "" { envVars = append(envVars, metadataDeviceEnvVar(osdProps.metadataDevice)) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index b465b082c..feaf07966 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -404,6 +404,11 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC } } + // Ceph expects initial weight as float value in tera-bytes units + if osdProps.storeConfig.InitialWeight != "" { + args = append(args, fmt.Sprintf("--osd-crush-initial-weight=%s", osdProps.storeConfig.InitialWeight)) + } + // If the OSD runs on PVC if osdProps.onPVC() { // add the PVC size to the pod spec so that if the size changes the OSD will be restarted and pick up the change