From a0a27bc42635bdc26f808c1d7962e72e3ca59798 Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Thu, 10 Dec 2020 17:11:48 -0700
Subject: [PATCH] ceph: apply deviceClass properly to new osds

The deviceClass property was being ignored when creating the
non-pvc OSDs. Now the deviceClass will be specified as a property
for individual devices, all devices on a node, or all OSDs in the
cluster, depending on the level where the config is applied in the
cluster CR.

Co-authored-by: shenjiatong <yshxxsjt715@gmail.com>
Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 cmd/rook/ceph/osd.go                            |  1 +
 pkg/daemon/ceph/osd/volume.go                   | 10 ++++++++--
 pkg/operator/ceph/cluster/osd/osd.go            |  3 +--
 pkg/operator/ceph/cluster/osd/provision_spec.go |  2 +-
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go
index 51ce194a2..874bdc13a 100644
--- a/cmd/rook/ceph/osd.go
+++ b/cmd/rook/ceph/osd.go
@@ -118,6 +118,7 @@ func addOSDConfigFlags(command *cobra.Command) {
 	command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)")
 	command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device")
 	command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt")
+	command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node")
 }
 
 func init() {
diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go
index 390e97feb..d109fcda4 100644
--- a/pkg/daemon/ceph/osd/volume.go
+++ b/pkg/daemon/ceph/osd/volume.go
@@ -524,10 +524,16 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 					deviceArg,
 				}...)
 
-				if a.storeConfig.DeviceClass != "" {
+				// assign the device class specific to the device
+				deviceClass := device.Config.DeviceClass
+				if deviceClass == "" {
+					// fall back to the device class for all devices on the node
+					deviceClass = a.storeConfig.DeviceClass
+				}
+				if deviceClass != "" {
 					immediateExecuteArgs = append(immediateExecuteArgs, []string{
 						crushDeviceClassFlag,
-						a.storeConfig.DeviceClass,
+						deviceClass,
 					}...)
 				}
 
diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go
index c407dbd6f..fd5d12bce 100644
--- a/pkg/operator/ceph/cluster/osd/osd.go
+++ b/pkg/operator/ceph/cluster/osd/osd.go
@@ -139,7 +139,6 @@ type osdProperties struct {
 	tuneSlowDeviceClass bool
 	tuneFastDeviceClass bool
 	schedulerName       string
-	crushDeviceClass    string
 	encrypted           bool
 	deviceSetName       string
 	// Drive Groups which apply to the node
@@ -268,11 +267,11 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) {
 			placement:        volume.Placement,
 			preparePlacement: volume.PreparePlacement,
 			portable:         volume.Portable,
-			crushDeviceClass: volume.CrushDeviceClass,
 			schedulerName:    volume.SchedulerName,
 			encrypted:        volume.Encrypted,
 			deviceSetName:    volume.Name,
 		}
+		osdProps.storeConfig.DeviceClass = volume.CrushDeviceClass
 
 		logger.Debugf("osdProps are %+v", osdProps)
 
diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go
index c52fbc435..2fc26f6f1 100644
--- a/pkg/operator/ceph/cluster/osd/provision_spec.go
+++ b/pkg/operator/ceph/cluster/osd/provision_spec.go
@@ -233,6 +233,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
 		envVars = append(envVars, deviceFilterEnvVar("all"))
 	}
 	envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()})
+	envVars = append(envVars, crushDeviceClassEnvVar(osdProps.storeConfig.DeviceClass))
 
 	if osdProps.metadataDevice != "" {
 		envVars = append(envVars, metadataDeviceEnvVar(osdProps.metadataDevice))
@@ -281,7 +282,6 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
 		}
 		envVars = append(envVars, dataDevicesEnvVar(string(marshalledDevices)))
 		envVars = append(envVars, pvcBackedOSDEnvVar("true"))
-		envVars = append(envVars, crushDeviceClassEnvVar(osdProps.crushDeviceClass))
 		envVars = append(envVars, encryptedDeviceEnvVar(osdProps.encrypted))
 		envVars = append(envVars, pvcNameEnvVar(osdProps.pvc.ClaimName))