Merge pull request #3696 from mvollman/db-devices-n-size

ceph: add db-devices flag and fix DatabaseSizeMB

PendingReleaseNotes.md

@@ -14,6 +14,8 @@ an example usage
 - Allow `metadataDevice` to be set per OSD device in the device specific `config` section.
 - [YugabyteDB](https://www.yugabyte.com/) is now supported by Rook with a new operator. You can deploy, configure and manage instances of this high-performance distributed SQL database. Create an instance of the new `ybcluster.yugabytedb.rook.io` custom resource to easily deploy a cluster of YugabyteDB Database. Checkout its [user guide](Documentation/yugabytedb.md) to get started with YugabyteDB.
 - Git tags can be added for alpha, beta, or rc releases. For example: v1.1.0-alpha.0
+- Added `deviceClass` to the per OSD device specific `config` section for setting custom crush device class per OSD.
+- Use `--db-devices` with Ceph 14.2.1 and newer clusters to explicitly set `metadataDevice` per OSD.
 
 ### Ceph
 

cmd/rook/ceph/osd.go

@@ -275,9 +275,9 @@ func commonOSDInit(cmd *cobra.Command) {
 // Parse the devices, which are comma separated. A colon indicates a non-default number of osds per device
 // or a non collocated metadata device.
 // For example, one osd will be created on each of sda and sdb, with 5 osds on the nvme01 device.
-//   sda:1,sdb:1,nvme01:5
+//   sda:1:::,sdb:1:::,nvme01:5:::
 // For example, 3 osds will use sdb SSD for db and 3 osds will use sdc SSD for db.
-//   sdd:1:sdb,sde:1:sdb,sdf:1:sdb,sdg:1:sdc,sdh:1:sdc,sdi:1:sdc
+//   sdd:1:::sdb,sde:1:::sdb,sdf:1:::sdb,sdg:1:::sdc,sdh:1:::sdc,sdi:1:::sdc
 func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) {
 	var result []osddaemon.DesiredDevice
 	parsed := strings.Split(devices, ",")
@@ -294,8 +294,18 @@ func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) {
 			}
 			d.OSDsPerDevice = count
 		}
-		if len(parts) > 2 {
-			d.MetadataDevice = parts[2]
+		if len(parts) > 2 && parts[2] != "" {
+			size, err := strconv.Atoi(parts[2])
+			if err != nil {
+				return nil, fmt.Errorf("error DatabaseSizeMB (%s) to int. %+v", parts[2], err)
+			}
+			d.DatabaseSizeMB = size
+		}
+		if len(parts) > 3 && parts[3] != "" {
+			d.DeviceClass = parts[3]
+		}
+		if len(parts) > 4 {
+			d.MetadataDevice = parts[4]
 		}
 		result = append(result, d)
 	}

cmd/rook/ceph/osd_test.go

@@ -49,8 +49,8 @@ func TestParseDesiredDevices(t *testing.T) {
 	assert.Nil(t, result)
 	assert.NotNil(t, err)
 
-	// metadataDevice and OSDsPerDevice
-	devices = "sdd:1:sdb,sde:1:sdb,sdf:1:sdc,sdg:1:sdc"
+	// OSDsPerDevice, metadataDevice, databaseSizeMB and deviceClass
+	devices = "sdd:1:2048::sdb,sde:1:::sdb,sdf:1:::sdc,sdg:1::tst:sdc"
 	result, err = parseDevices(devices)
 	assert.Equal(t, "sdd", result[0].Name)
 	assert.Equal(t, "sde", result[1].Name)
@@ -60,6 +60,14 @@ func TestParseDesiredDevices(t *testing.T) {
 	assert.Equal(t, 1, result[1].OSDsPerDevice)
 	assert.Equal(t, 1, result[2].OSDsPerDevice)
 	assert.Equal(t, 1, result[3].OSDsPerDevice)
+	assert.Equal(t, 2048, result[0].DatabaseSizeMB)
+	assert.Equal(t, 0, result[1].DatabaseSizeMB)
+	assert.Equal(t, 0, result[2].DatabaseSizeMB)
+	assert.Equal(t, 0, result[3].DatabaseSizeMB)
+	assert.Equal(t, "", result[0].DeviceClass)
+	assert.Equal(t, "", result[1].DeviceClass)
+	assert.Equal(t, "", result[2].DeviceClass)
+	assert.Equal(t, "tst", result[3].DeviceClass)
 	assert.Equal(t, "sdb", result[0].MetadataDevice)
 	assert.Equal(t, "sdb", result[1].MetadataDevice)
 	assert.Equal(t, "sdc", result[2].MetadataDevice)

pkg/daemon/ceph/osd/device.go

@@ -76,6 +76,8 @@ type DesiredDevice struct {
 	Name           string
 	OSDsPerDevice  int
 	MetadataDevice string
+	DatabaseSizeMB int
+	DeviceClass    string
 	IsFilter       bool
 }
 

pkg/daemon/ceph/osd/volume.go

@@ -29,6 +29,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/util/display"
 	"github.com/rook/rook/pkg/util/exec"
 	"github.com/rook/rook/pkg/util/sys"
@@ -37,12 +38,14 @@ import (
 var cephConfigDir = "/var/lib/ceph"
 
 const (
-	osdsPerDeviceFlag   = "--osds-per-device"
-	encryptedFlag       = "--dmcrypt"
-	databaseSizeFlag    = "--block-db-size"
-	cephVolumeCmd       = "ceph-volume"
-	cephVolumeMinDBSize = 1024 // 1GB
-	lvmConfPath         = "/etc/lvm/lvm.conf"
+	osdsPerDeviceFlag    = "--osds-per-device"
+	crushDeviceClassFlag = "--crush-device-class"
+	encryptedFlag        = "--dmcrypt"
+	databaseSizeFlag     = "--block-db-size"
+	dbDeviceFlag         = "--db-devices"
+	cephVolumeCmd        = "ceph-volume"
+	cephVolumeMinDBSize  = 1024 // 1GB
+	lvmConfPath          = "/etc/lvm/lvm.conf"
 )
 
 func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, error) {
@@ -167,20 +170,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 	osdsPerDeviceCount := sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice)
 	batchArgs := baseArgs
 
-	if a.storeConfig.StoreType == config.Bluestore && a.storeConfig.DatabaseSizeMB > 0 {
-		if a.storeConfig.DatabaseSizeMB < cephVolumeMinDBSize {
-			// ceph-volume will convert this value to ?G. It needs to be > 1G to invoke lvcreate.
-			logger.Infof("skipping databaseSizeMB setting. For it should be larger than %dMB.", cephVolumeMinDBSize)
-		} else {
-			batchArgs = append(batchArgs, []string{
-				databaseSizeFlag,
-				// ceph-volume takes in this value in bytes
-				strconv.FormatUint(display.MbTob(uint64(a.storeConfig.DatabaseSizeMB)), 10),
-			}...)
-		}
-	}
-
-	metadataDevices := make(map[string][]string)
+	metadataDevices := make(map[string]map[string]string)
 	for name, device := range devices.Entries {
 		if device.LegacyPartitionsFound {
 			logger.Infof("skipping device %s configured with legacy rook osd", name)
@@ -190,10 +180,12 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 		if device.Data == -1 {
 			logger.Infof("configuring new device %s", name)
 			deviceArg := path.Join("/dev", name)
+
 			deviceOSDCount := osdsPerDeviceCount
 			if device.Config.OSDsPerDevice > 1 {
 				deviceOSDCount = sanitizeOSDsPerDevice(device.Config.OSDsPerDevice)
 			}
+
 			if a.metadataDevice != "" || device.Config.MetadataDevice != "" {
 				// When mixed hdd/ssd devices are given, ceph-volume configures db lv on the ssd.
 				// the device will be configured as a batch at the end of the method
@@ -203,16 +195,33 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 				}
 				logger.Infof("using %s as metadataDevice for device %s and let ceph-volume lvm batch decide how to create volumes", md, deviceArg)
 				if _, ok := metadataDevices[md]; ok {
-					metadataDevices[md] = append(metadataDevices[md], deviceArg)
 					// Fail when two devices using the same metadata device have different values for osdsPerDevice
-					if deviceOSDCount != metadataDevices[md][1] {
-						return fmt.Errorf("metadataDevice (%s) has more than 1 osdsPerDevice value set: %s != %s", md, deviceOSDCount, metadataDevices[md][1])
+					metadataDevices[md]["devices"] += " " + deviceArg
+					if deviceOSDCount != metadataDevices[md]["osdsperdevice"] {
+						return fmt.Errorf("metadataDevice (%s) has more than 1 osdsPerDevice value set: %s != %s", md, deviceOSDCount, metadataDevices[md]["osdsperdevice"])
 					}
 				} else {
-					metadataDevices[md] = []string{
-						osdsPerDeviceFlag,
-						deviceOSDCount,
-						deviceArg,
+					metadataDevices[md] = make(map[string]string)
+					metadataDevices[md]["osdsperdevice"] = deviceOSDCount
+					if device.Config.DeviceClass != "" {
+						metadataDevices[md]["deviceclass"] = device.Config.DeviceClass
+					}
+					metadataDevices[md]["devices"] = deviceArg
+				}
+				deviceDBSizeMB := getDatabaseSize(a.storeConfig.DatabaseSizeMB, device.Config.DatabaseSizeMB)
+				if storeFlag == "--bluestore" && deviceDBSizeMB > 0 {
+					if deviceDBSizeMB < cephVolumeMinDBSize {
+						// ceph-volume will convert this value to ?G. It needs to be > 1G to invoke lvcreate.
+						logger.Infof("skipping databaseSizeMB setting (%d). For it should be larger than %dMB.", deviceDBSizeMB, cephVolumeMinDBSize)
+					} else {
+						dbSizeString := strconv.FormatUint(display.MbTob(uint64(deviceDBSizeMB)), 10)
+						if _, ok := metadataDevices[md]["databasesizemb"]; ok {
+							if metadataDevices[md]["databasesizemb"] != dbSizeString {
+								return fmt.Errorf("metadataDevice (%s) has more than 1 databaseSizeMB value set: %s != %s", md, metadataDevices[md]["databasesizemb"], dbSizeString)
+							}
+						} else {
+							metadataDevices[md]["databasesizemb"] = dbSizeString
+						}
 					}
 				}
 			} else {
@@ -222,6 +231,13 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 					deviceArg,
 				}...)
 
+				if device.Config.DeviceClass != "" {
+					immediateExecuteArgs = append(immediateExecuteArgs, []string{
+						crushDeviceClassFlag,
+						device.Config.DeviceClass,
+					}...)
+				}
+
 				// Reporting
 				immediateReportArgs := append(immediateExecuteArgs, []string{
 					"--report",
@@ -245,10 +261,37 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 		}
 	}
 
-	for md, devs := range metadataDevices {
+	for md, conf := range metadataDevices {
 
-		mdArgs := append(batchArgs, devs...)
-		mdArgs = append(mdArgs, path.Join("/dev", md))
+		mdArgs := batchArgs
+		if _, ok := conf["osdsperdevice"]; ok {
+			mdArgs = append(mdArgs, []string{
+				osdsPerDeviceFlag,
+				conf["osdsperdevice"],
+			}...)
+		}
+		if _, ok := conf["deviceclass"]; ok {
+			mdArgs = append(mdArgs, []string{
+				crushDeviceClassFlag,
+				conf["deviceclass"],
+			}...)
+		}
+		if _, ok := conf["databasesizemb"]; ok {
+			mdArgs = append(mdArgs, []string{
+				databaseSizeFlag,
+				conf["databasesizemb"],
+			}...)
+		}
+		mdArgs = append(mdArgs, strings.Split(conf["devices"], " ")...)
+
+		if a.cluster.CephVersion.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 1}) {
+			mdArgs = append(mdArgs, []string{
+				dbDeviceFlag,
+				path.Join("/dev", md),
+			}...)
+		} else {
+			mdArgs = append(mdArgs, path.Join("/dev", md))
+		}
 
 		// Reporting
 		reportArgs := append(mdArgs, []string{
@@ -289,6 +332,13 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 	return nil
 }
 
+func getDatabaseSize(globalSize int, deviceSize int) int {
+	if deviceSize > 0 {
+		globalSize = deviceSize
+	}
+	return globalSize
+}
+
 func sanitizeOSDsPerDevice(count int) string {
 	if count < 1 {
 		count = 1

pkg/daemon/ceph/osd/volume_test.go

@@ -306,3 +306,8 @@ func TestSanitizeOSDsPerDevice(t *testing.T) {
 	assert.Equal(t, "1", sanitizeOSDsPerDevice(1))
 	assert.Equal(t, "2", sanitizeOSDsPerDevice(2))
 }
+
+func TestGetDatabaseSize(t *testing.T) {
+	assert.Equal(t, 0, getDatabaseSize(0, 0))
+	assert.Equal(t, 2048, getDatabaseSize(4096, 2048))
+}

pkg/operator/ceph/cluster/osd/config/config.go

@@ -44,6 +44,7 @@ const (
 	OSDsPerDeviceKey   = "osdsPerDevice"
 	EncryptedDeviceKey = "encryptedDevice"
 	MetadataDeviceKey  = "metadataDevice"
+	DeviceClassKey     = "deviceClass"
 )
 
 type StoreConfig struct {
@@ -53,6 +54,7 @@ type StoreConfig struct {
 	JournalSizeMB   int    `json:"journalSizeMB,omitempty"`
 	OSDsPerDevice   int    `json:"osdsPerDevice,omitempty"`
 	EncryptedDevice bool   `json:"encryptedDevice,omitempty"`
+	DeviceClass     string `json:"deviceClass,omitempty"`
 }
 
 func ToStoreConfig(config map[string]string) StoreConfig {
@@ -71,6 +73,8 @@ func ToStoreConfig(config map[string]string) StoreConfig {
 			storeConfig.OSDsPerDevice = convertToIntIgnoreErr(v)
 		case EncryptedDeviceKey:
 			storeConfig.EncryptedDevice = (v == "true")
+		case DeviceClassKey:
+			storeConfig.DeviceClass = v
 		}
 	}
 

pkg/operator/ceph/cluster/osd/spec.go

@@ -555,9 +555,23 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
 			} else {
 				devSuffix += ":1"
 			}
+			if databaseSizeMB, ok := device.Config[config.DatabaseSizeMBKey]; ok {
+				logger.Infof("osd %s requested with DB size %sMB (node %s)", device.Name, databaseSizeMB, osdProps.crushHostname)
+				devSuffix += ":" + databaseSizeMB
+			} else {
+				devSuffix += ":"
+			}
+			if deviceClass, ok := device.Config[config.DeviceClassKey]; ok {
+				logger.Infof("osd %s requested with deviceClass %s (node %s)", device.Name, deviceClass, osdProps.crushHostname)
+				devSuffix += ":" + deviceClass
+			} else {
+				devSuffix += ":"
+			}
 			if md, ok := device.Config[config.MetadataDeviceKey]; ok {
 				logger.Infof("osd %s requested with metadataDevice %s (node %s)", device.Name, md, osdProps.crushHostname)
 				devSuffix += ":" + md
+			} else {
+				devSuffix += ":"
 			}
 			deviceNames[i] = device.Name + devSuffix
 		}
@@ -570,6 +584,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
 		envVars = append(envVars, deviceFilterEnvVar("all"))
 		devMountNeeded = true
 	}
+	envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()})
 
 	if osdProps.metadataDevice != "" {
 		envVars = append(envVars, metadataDeviceEnvVar(osdProps.metadataDevice))

pkg/operator/ceph/cluster/osd/spec_test.go

@@ -38,7 +38,7 @@ import (
 )
 
 func TestPodContainer(t *testing.T) {
-	cluster := &Cluster{Namespace: "myosd", rookVersion: "23", cephVersion: cephv1.CephVersionSpec{}}
+	cluster := &Cluster{Namespace: "myosd", rookVersion: "23", cephVersion: cephv1.CephVersionSpec{}, clusterInfo: &cephconfig.ClusterInfo{}}
 	osdProps := osdProperties{
 		crushHostname: "node",
 		devices:       []rookalpha.Device{},