osd: handle global or node-local device class configuration correctly

Rook uses global or node-local device class configuration if device-level
configurations doesn't exist. However, after introducing device-class
level resource configuration, rook set the default value of device class.
So global or node-level device class configuration has never been used
after that.

Closes: https://github.com/rook/rook/issues/11871
Closes: https://github.com/rook/rook/issues/11826

Signed-off-by: Satoru Takeuchi <satoru.takeuchi@gmail.com>
(cherry picked from commit 3e34ebeff8)

pkg/daemon/ceph/osd/daemon.go

@@ -531,19 +531,7 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd
 
 				if matched {
 					matchedDevice = desiredDevice
-					if matchedDevice.DeviceClass == "" {
-						classNotSet := true
-						if agent.pvcBacked {
-							crushDeviceClass := os.Getenv(oposd.CrushDeviceClassVarName)
-							if crushDeviceClass != "" {
-								matchedDevice.DeviceClass = crushDeviceClass
-								classNotSet = false
-							}
-						}
-						if classNotSet {
-							matchedDevice.DeviceClass = sys.GetDiskDeviceClass(device)
-						}
-					}
+					matchedDevice.UpdateDeviceClass(agent, device)
 					break
 				}
 			}

pkg/daemon/ceph/osd/device.go

@@ -18,7 +18,9 @@ package osd
 
 import (
 	"encoding/json"
+	"os"
 
+	oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd"
 	"github.com/rook/rook/pkg/util/sys"
 )
 
@@ -67,3 +69,29 @@ func (m *DeviceOsdMapping) String() string {
 	b, _ := json.Marshal(m)
 	return string(b)
 }
+
+func (d *DesiredDevice) UpdateDeviceClass(agent *OsdAgent, device *sys.LocalDisk) {
+	// Rook sets the storage class of a device with the following priority.
+	//
+	// 1. the device-level configuration
+	// 2. the global or node-local configuration
+	// 3. the default value estimated from sysfs.
+	if d.DeviceClass != "" {
+		return
+	}
+
+	if agent.pvcBacked {
+		crushDeviceClass := os.Getenv(oposd.CrushDeviceClassVarName)
+		if crushDeviceClass != "" {
+			d.DeviceClass = crushDeviceClass
+			return
+		}
+	} else {
+		if agent.storeConfig.DeviceClass != "" {
+			d.DeviceClass = agent.storeConfig.DeviceClass
+			return
+		}
+	}
+
+	d.DeviceClass = sys.GetDiskDeviceClass(device)
+}

pkg/daemon/ceph/osd/device_test.go

@@ -23,7 +23,9 @@ import (
 
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
+	"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
+	"github.com/rook/rook/pkg/util/sys"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -47,3 +49,36 @@ func TestOSDBootstrap(t *testing.T) {
 	assert.NotEqual(t, -1, strings.Index(string(contents), "key = mysecurekey"))
 	assert.NotEqual(t, -1, strings.Index(string(contents), "caps mon = \"allow profile bootstrap-osd\""))
 }
+
+func TestUpdateDeviceClass(t *testing.T) {
+	d := &DesiredDevice{}
+	agent := &OsdAgent{}
+	disk := &sys.LocalDisk{}
+
+	d.DeviceClass = "test"
+	d.UpdateDeviceClass(agent, disk)
+	assert.Equal(t, "test", d.DeviceClass)
+
+	d.DeviceClass = ""
+	agent.pvcBacked = true
+	t.Setenv(osd.CrushDeviceClassVarName, "test")
+	d.UpdateDeviceClass(agent, disk)
+	assert.Equal(t, "test", d.DeviceClass)
+
+	d.DeviceClass = ""
+	t.Setenv(osd.CrushDeviceClassVarName, "")
+	d.UpdateDeviceClass(agent, disk)
+	t.Log(d)
+	t.Log(disk)
+	assert.Equal(t, "ssd", d.DeviceClass)
+
+	d.DeviceClass = ""
+	agent.pvcBacked = false
+	d.UpdateDeviceClass(agent, disk)
+	assert.Equal(t, "ssd", d.DeviceClass)
+
+	d.DeviceClass = ""
+	agent.storeConfig.DeviceClass = "test"
+	d.UpdateDeviceClass(agent, disk)
+	assert.Equal(t, "test", d.DeviceClass)
+}

pkg/util/sys/device_test.go

@@ -196,3 +196,13 @@ func TestListDevicesChildListDevicesChild(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 3, len(child))
 }
+
+func TestGetDiskDeviceClass(t *testing.T) {
+	d := &LocalDisk{}
+	assert.Equal(t, "ssd", GetDiskDeviceClass(d))
+	d.Rotational = true
+	assert.Equal(t, "hdd", GetDiskDeviceClass(d))
+	d.Rotational = false
+	d.RealPath = "nvme"
+	assert.Equal(t, "nvme", GetDiskDeviceClass(d))
+}