From 9766e9b8fb2c03ee2cd02c8dade569c1d41cbb2e Mon Sep 17 00:00:00 2001 From: Shachar Sharon Date: Tue, 30 Mar 2021 11:11:41 +0300 Subject: [PATCH] ceph: allow passing 'osd-crush-initial-weight' Ceph support the option '--osd-crush-initial-weight' upon OSD start, which sets an explicit weight (in TiB units) to specific OSD. Allow passing this option all the way from the user (similar to 'DeviceClass'), for the special case where end users wants it cluster to have non-even balance over specific OSDs (e.g., one of the OSDs is placed over a partition alongside OS-partition). ROOK issue: https://github.com/rook/rook/issues/7448 Signed-off-by: Shachar Sharon --- .../charts/rook-ceph/templates/resources.yaml | 4 +++ cluster/examples/kubernetes/ceph/crds.yaml | 4 +++ cmd/rook/ceph/osd.go | 2 ++ pkg/apis/rook.io/v1/types.go | 4 +++ pkg/daemon/ceph/osd/device.go | 1 + .../ceph/cluster/osd/config/config.go | 4 +++ pkg/operator/ceph/cluster/osd/deviceSet.go | 4 +++ .../ceph/cluster/osd/deviceset_test.go | 34 +++++++++++++++++++ pkg/operator/ceph/cluster/osd/envs.go | 5 +++ pkg/operator/ceph/cluster/osd/osd.go | 2 ++ .../ceph/cluster/osd/provision_spec.go | 1 + pkg/operator/ceph/cluster/osd/spec.go | 5 +++ 12 files changed, 70 insertions(+) diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 277afe08e..5c736f7c2 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -3160,6 +3160,10 @@ spec: crushDeviceClass: description: CrushDeviceClass represents the crush device class for an OSD type: string + crushInitialWeight: + description: CrushInitialWeight represents initial OSD weight in TiB units + pattern: ^([0-9]*[.])?[0-9]$ + type: string encrypted: description: Whether to encrypt the deviceSet type: boolean diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 2b779d25f..69ce3afe7 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -3162,6 +3162,10 @@ spec: crushDeviceClass: description: CrushDeviceClass represents the crush device class for an OSD type: string + crushInitialWeight: + description: CrushInitialWeight represents initial OSD weight in TiB units + pattern: ^([0-9]*[.])?[0-9]$ + type: string encrypted: description: Whether to encrypt the deviceSet type: boolean diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 203cdedda..38071f4d9 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -118,6 +118,7 @@ func addOSDConfigFlags(command *cobra.Command) { command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device") command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt") command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node") + command.Flags().StringVar(&cfg.storeConfig.InitialWeight, "osd-crush-initial-weight", "", "The initial weight of OSD in TiB units") } func init() { @@ -303,6 +304,7 @@ func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) { d.OSDsPerDevice = cd.StoreConfig.OSDsPerDevice d.DatabaseSizeMB = cd.StoreConfig.DatabaseSizeMB d.DeviceClass = cd.StoreConfig.DeviceClass + d.InitialWeight = cd.StoreConfig.InitialWeight d.MetadataDevice = cd.StoreConfig.MetadataDevice if d.OSDsPerDevice < 1 { diff --git a/pkg/apis/rook.io/v1/types.go b/pkg/apis/rook.io/v1/types.go index 054b2d601..40b47df85 100644 --- a/pkg/apis/rook.io/v1/types.go +++ b/pkg/apis/rook.io/v1/types.go @@ -217,6 +217,10 @@ type VolumeSource struct { // CrushDeviceClass represents the crush device class for an OSD // +optional CrushDeviceClass string `json:"crushDeviceClass,omitempty"` + // CrushInitialWeight represents initial OSD weight in TiB units + // +kubebuilder:validation:Pattern=`^([0-9]*[.])?[0-9]$` + // +optional + CrushInitialWeight string `json:"crushInitialWeight,omitempty"` // Size represents the size requested for the PVC Size string `json:"size"` // Resources requests/limits for the devices diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index d6bf8a7eb..29e937487 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -42,6 +42,7 @@ type DesiredDevice struct { MetadataDevice string DatabaseSizeMB int DeviceClass string + InitialWeight string IsFilter bool IsDevicePathFilter bool } diff --git a/pkg/operator/ceph/cluster/osd/config/config.go b/pkg/operator/ceph/cluster/osd/config/config.go index c706f4a1d..d0daa4dc1 100644 --- a/pkg/operator/ceph/cluster/osd/config/config.go +++ b/pkg/operator/ceph/cluster/osd/config/config.go @@ -29,6 +29,7 @@ const ( EncryptedDeviceKey = "encryptedDevice" MetadataDeviceKey = "metadataDevice" DeviceClassKey = "deviceClass" + InitialWeightKey = "initialWeight" ) // StoreConfig represents the configuration of an OSD on a device. @@ -39,6 +40,7 @@ type StoreConfig struct { EncryptedDevice bool `json:"encryptedDevice,omitempty"` MetadataDevice string `json:"metadataDevice,omitempty"` DeviceClass string `json:"deviceClass,omitempty"` + InitialWeight string `json:"initialWeight,omitempty"` } // NewStoreConfig returns a StoreConfig with proper defaults set. @@ -68,6 +70,8 @@ func ToStoreConfig(config map[string]string) StoreConfig { storeConfig.MetadataDevice = v case DeviceClassKey: storeConfig.DeviceClass = v + case InitialWeightKey: + storeConfig.InitialWeight = v } } diff --git a/pkg/operator/ceph/cluster/osd/deviceSet.go b/pkg/operator/ceph/cluster/osd/deviceSet.go index f73d672f0..ca4c2b795 100644 --- a/pkg/operator/ceph/cluster/osd/deviceSet.go +++ b/pkg/operator/ceph/cluster/osd/deviceSet.go @@ -96,6 +96,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic var dataSize string var crushDeviceClass string + var crushInitialWeight string typesFound := util.NewSet() for _, pvcTemplate := range deviceSet.VolumeClaimTemplates { if pvcTemplate.Name == "" { @@ -126,6 +127,8 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic dataSize = pvcSize.String() crushDeviceClass = pvcTemplate.Annotations["crushDeviceClass"] } + crushInitialWeight = pvcTemplate.Annotations["crushInitialWeight"] + pvcSources[pvcType] = v1.PersistentVolumeClaimVolumeSource{ ClaimName: pvc.GetName(), ReadOnly: false, @@ -145,6 +148,7 @@ func (c *Cluster) createDeviceSetPVCsForIndex(deviceSet rookv1.StorageClassDevic TuneFastDeviceClass: deviceSet.TuneFastDeviceClass, SchedulerName: deviceSet.SchedulerName, CrushDeviceClass: crushDeviceClass, + CrushInitialWeight: crushInitialWeight, Encrypted: deviceSet.Encrypted, } } diff --git a/pkg/operator/ceph/cluster/osd/deviceset_test.go b/pkg/operator/ceph/cluster/osd/deviceset_test.go index 0d06c00bc..8ae0582f9 100644 --- a/pkg/operator/ceph/cluster/osd/deviceset_test.go +++ b/pkg/operator/ceph/cluster/osd/deviceset_test.go @@ -269,3 +269,37 @@ func TestUpdatePVCSize(t *testing.T) { assert.True(t, ok) assert.Equal(t, "6Gi", result.String()) } + +func TestPrepareDeviceSetsWithCrushParams(t *testing.T) { + ctx := context.TODO() + clientset := testexec.New(t, 1) + context := &clusterd.Context{ + Clientset: clientset, + } + deviceSet := rookv1.StorageClassDeviceSet{ + Name: "datawithcrushparams1", + Count: 1, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{testVolumeClaim("testwithcrushparams1")}, + SchedulerName: "custom-scheduler", + } + deviceSet.VolumeClaimTemplates[0].Annotations = map[string]string{"crushDeviceClass": "ssd", "crushInitialWeight": "0.75"} + + spec := cephv1.ClusterSpec{ + Storage: rookv1.StorageScopeSpec{StorageClassDeviceSets: []rookv1.StorageClassDeviceSet{deviceSet}}, + } + cluster := &Cluster{ + context: context, + clusterInfo: client.AdminClusterInfo("testns"), + spec: spec, + } + + config := newProvisionErrors() + volumeSources := cluster.prepareStorageClassDeviceSets(config) + assert.Equal(t, 1, len(volumeSources)) + assert.Equal(t, volumeSources[0].CrushDeviceClass, "ssd") + assert.Equal(t, volumeSources[0].CrushInitialWeight, "0.75") + + pvcs, err := clientset.CoreV1().PersistentVolumeClaims(cluster.clusterInfo.Namespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + assert.Equal(t, 1, len(pvcs.Items)) +} diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index 677fcea76..b8d9b04b9 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -45,6 +45,7 @@ const ( cvModeVarName = "ROOK_CV_MODE" lvBackedPVVarName = "ROOK_LV_BACKED_PV" CrushDeviceClassVarName = "ROOK_OSD_CRUSH_DEVICE_CLASS" + CrushInitialWeightVarName = "ROOK_OSD_CRUSH_INITIAL_WEIGHT" CrushRootVarName = "ROOK_CRUSHMAP_ROOT" tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES" ) @@ -159,6 +160,10 @@ func crushDeviceClassEnvVar(crushDeviceClass string) v1.EnvVar { return v1.EnvVar{Name: CrushDeviceClassVarName, Value: crushDeviceClass} } +func crushInitialWeightEnvVar(crushInitialWeight string) v1.EnvVar { + return v1.EnvVar{Name: CrushInitialWeightVarName, Value: crushInitialWeight} +} + func encryptedDeviceEnvVar(encryptedDevice bool) v1.EnvVar { return v1.EnvVar{Name: EncryptedDeviceEnvVarName, Value: strconv.FormatBool(encryptedDevice)} } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index ac3d3ac23..fa6d378f5 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -363,6 +363,8 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) { encrypted: volumeSource.Encrypted, deviceSetName: volumeSource.Name, } + osdProps.storeConfig.InitialWeight = volumeSource.CrushInitialWeight + // If OSD isn't portable, we're getting the host name either from the osd deployment that was already initialized // or from the osd prepare job from initial creation. if !volumeSource.Portable { diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 2035f1ab3..e1f0f50ae 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -229,6 +229,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun } envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()}) envVars = append(envVars, crushDeviceClassEnvVar(osdProps.storeConfig.DeviceClass)) + envVars = append(envVars, crushInitialWeightEnvVar(osdProps.storeConfig.InitialWeight)) if osdProps.metadataDevice != "" { envVars = append(envVars, metadataDeviceEnvVar(osdProps.metadataDevice)) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index b465b082c..feaf07966 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -404,6 +404,11 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC } } + // Ceph expects initial weight as float value in tera-bytes units + if osdProps.storeConfig.InitialWeight != "" { + args = append(args, fmt.Sprintf("--osd-crush-initial-weight=%s", osdProps.storeConfig.InitialWeight)) + } + // If the OSD runs on PVC if osdProps.onPVC() { // add the PVC size to the pod spec so that if the size changes the OSD will be restarted and pick up the change