diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 8b8ff3288..813412d9a 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -14,6 +14,8 @@ an example usage - Allow `metadataDevice` to be set per OSD device in the device specific `config` section. - [YugabyteDB](https://www.yugabyte.com/) is now supported by Rook with a new operator. You can deploy, configure and manage instances of this high-performance distributed SQL database. Create an instance of the new `ybcluster.yugabytedb.rook.io` custom resource to easily deploy a cluster of YugabyteDB Database. Checkout its [user guide](Documentation/yugabytedb.md) to get started with YugabyteDB. - Git tags can be added for alpha, beta, or rc releases. For example: v1.1.0-alpha.0 +- Added `deviceClass` to the per OSD device specific `config` section for setting custom crush device class per OSD. +- Use `--db-devices` with Ceph 14.2.1 and newer clusters to explicitly set `metadataDevice` per OSD. ### Ceph diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 9795d9ce6..3bd132950 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -275,9 +275,9 @@ func commonOSDInit(cmd *cobra.Command) { // Parse the devices, which are comma separated. A colon indicates a non-default number of osds per device // or a non collocated metadata device. // For example, one osd will be created on each of sda and sdb, with 5 osds on the nvme01 device. -// sda:1,sdb:1,nvme01:5 +// sda:1:::,sdb:1:::,nvme01:5::: // For example, 3 osds will use sdb SSD for db and 3 osds will use sdc SSD for db. -// sdd:1:sdb,sde:1:sdb,sdf:1:sdb,sdg:1:sdc,sdh:1:sdc,sdi:1:sdc +// sdd:1:::sdb,sde:1:::sdb,sdf:1:::sdb,sdg:1:::sdc,sdh:1:::sdc,sdi:1:::sdc func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) { var result []osddaemon.DesiredDevice parsed := strings.Split(devices, ",") @@ -294,8 +294,18 @@ func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) { } d.OSDsPerDevice = count } - if len(parts) > 2 { - d.MetadataDevice = parts[2] + if len(parts) > 2 && parts[2] != "" { + size, err := strconv.Atoi(parts[2]) + if err != nil { + return nil, fmt.Errorf("error DatabaseSizeMB (%s) to int. %+v", parts[2], err) + } + d.DatabaseSizeMB = size + } + if len(parts) > 3 && parts[3] != "" { + d.DeviceClass = parts[3] + } + if len(parts) > 4 { + d.MetadataDevice = parts[4] } result = append(result, d) } diff --git a/cmd/rook/ceph/osd_test.go b/cmd/rook/ceph/osd_test.go index 3188a1748..16433425c 100644 --- a/cmd/rook/ceph/osd_test.go +++ b/cmd/rook/ceph/osd_test.go @@ -49,8 +49,8 @@ func TestParseDesiredDevices(t *testing.T) { assert.Nil(t, result) assert.NotNil(t, err) - // metadataDevice and OSDsPerDevice - devices = "sdd:1:sdb,sde:1:sdb,sdf:1:sdc,sdg:1:sdc" + // OSDsPerDevice, metadataDevice, databaseSizeMB and deviceClass + devices = "sdd:1:2048::sdb,sde:1:::sdb,sdf:1:::sdc,sdg:1::tst:sdc" result, err = parseDevices(devices) assert.Equal(t, "sdd", result[0].Name) assert.Equal(t, "sde", result[1].Name) @@ -60,6 +60,14 @@ func TestParseDesiredDevices(t *testing.T) { assert.Equal(t, 1, result[1].OSDsPerDevice) assert.Equal(t, 1, result[2].OSDsPerDevice) assert.Equal(t, 1, result[3].OSDsPerDevice) + assert.Equal(t, 2048, result[0].DatabaseSizeMB) + assert.Equal(t, 0, result[1].DatabaseSizeMB) + assert.Equal(t, 0, result[2].DatabaseSizeMB) + assert.Equal(t, 0, result[3].DatabaseSizeMB) + assert.Equal(t, "", result[0].DeviceClass) + assert.Equal(t, "", result[1].DeviceClass) + assert.Equal(t, "", result[2].DeviceClass) + assert.Equal(t, "tst", result[3].DeviceClass) assert.Equal(t, "sdb", result[0].MetadataDevice) assert.Equal(t, "sdb", result[1].MetadataDevice) assert.Equal(t, "sdc", result[2].MetadataDevice) diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index 9eda86b72..7380f9853 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -76,6 +76,8 @@ type DesiredDevice struct { Name string OSDsPerDevice int MetadataDevice string + DatabaseSizeMB int + DeviceClass string IsFilter bool } diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index b84491a75..af6e419e3 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -29,6 +29,7 @@ import ( "github.com/rook/rook/pkg/clusterd" oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/util/display" "github.com/rook/rook/pkg/util/exec" "github.com/rook/rook/pkg/util/sys" @@ -37,12 +38,14 @@ import ( var cephConfigDir = "/var/lib/ceph" const ( - osdsPerDeviceFlag = "--osds-per-device" - encryptedFlag = "--dmcrypt" - databaseSizeFlag = "--block-db-size" - cephVolumeCmd = "ceph-volume" - cephVolumeMinDBSize = 1024 // 1GB - lvmConfPath = "/etc/lvm/lvm.conf" + osdsPerDeviceFlag = "--osds-per-device" + crushDeviceClassFlag = "--crush-device-class" + encryptedFlag = "--dmcrypt" + databaseSizeFlag = "--block-db-size" + dbDeviceFlag = "--db-devices" + cephVolumeCmd = "ceph-volume" + cephVolumeMinDBSize = 1024 // 1GB + lvmConfPath = "/etc/lvm/lvm.conf" ) func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, error) { @@ -167,20 +170,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO osdsPerDeviceCount := sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice) batchArgs := baseArgs - if a.storeConfig.StoreType == config.Bluestore && a.storeConfig.DatabaseSizeMB > 0 { - if a.storeConfig.DatabaseSizeMB < cephVolumeMinDBSize { - // ceph-volume will convert this value to ?G. It needs to be > 1G to invoke lvcreate. - logger.Infof("skipping databaseSizeMB setting. For it should be larger than %dMB.", cephVolumeMinDBSize) - } else { - batchArgs = append(batchArgs, []string{ - databaseSizeFlag, - // ceph-volume takes in this value in bytes - strconv.FormatUint(display.MbTob(uint64(a.storeConfig.DatabaseSizeMB)), 10), - }...) - } - } - - metadataDevices := make(map[string][]string) + metadataDevices := make(map[string]map[string]string) for name, device := range devices.Entries { if device.LegacyPartitionsFound { logger.Infof("skipping device %s configured with legacy rook osd", name) @@ -190,10 +180,12 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO if device.Data == -1 { logger.Infof("configuring new device %s", name) deviceArg := path.Join("/dev", name) + deviceOSDCount := osdsPerDeviceCount if device.Config.OSDsPerDevice > 1 { deviceOSDCount = sanitizeOSDsPerDevice(device.Config.OSDsPerDevice) } + if a.metadataDevice != "" || device.Config.MetadataDevice != "" { // When mixed hdd/ssd devices are given, ceph-volume configures db lv on the ssd. // the device will be configured as a batch at the end of the method @@ -203,16 +195,33 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO } logger.Infof("using %s as metadataDevice for device %s and let ceph-volume lvm batch decide how to create volumes", md, deviceArg) if _, ok := metadataDevices[md]; ok { - metadataDevices[md] = append(metadataDevices[md], deviceArg) // Fail when two devices using the same metadata device have different values for osdsPerDevice - if deviceOSDCount != metadataDevices[md][1] { - return fmt.Errorf("metadataDevice (%s) has more than 1 osdsPerDevice value set: %s != %s", md, deviceOSDCount, metadataDevices[md][1]) + metadataDevices[md]["devices"] += " " + deviceArg + if deviceOSDCount != metadataDevices[md]["osdsperdevice"] { + return fmt.Errorf("metadataDevice (%s) has more than 1 osdsPerDevice value set: %s != %s", md, deviceOSDCount, metadataDevices[md]["osdsperdevice"]) } } else { - metadataDevices[md] = []string{ - osdsPerDeviceFlag, - deviceOSDCount, - deviceArg, + metadataDevices[md] = make(map[string]string) + metadataDevices[md]["osdsperdevice"] = deviceOSDCount + if device.Config.DeviceClass != "" { + metadataDevices[md]["deviceclass"] = device.Config.DeviceClass + } + metadataDevices[md]["devices"] = deviceArg + } + deviceDBSizeMB := getDatabaseSize(a.storeConfig.DatabaseSizeMB, device.Config.DatabaseSizeMB) + if storeFlag == "--bluestore" && deviceDBSizeMB > 0 { + if deviceDBSizeMB < cephVolumeMinDBSize { + // ceph-volume will convert this value to ?G. It needs to be > 1G to invoke lvcreate. + logger.Infof("skipping databaseSizeMB setting (%d). For it should be larger than %dMB.", deviceDBSizeMB, cephVolumeMinDBSize) + } else { + dbSizeString := strconv.FormatUint(display.MbTob(uint64(deviceDBSizeMB)), 10) + if _, ok := metadataDevices[md]["databasesizemb"]; ok { + if metadataDevices[md]["databasesizemb"] != dbSizeString { + return fmt.Errorf("metadataDevice (%s) has more than 1 databaseSizeMB value set: %s != %s", md, metadataDevices[md]["databasesizemb"], dbSizeString) + } + } else { + metadataDevices[md]["databasesizemb"] = dbSizeString + } } } } else { @@ -222,6 +231,13 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO deviceArg, }...) + if device.Config.DeviceClass != "" { + immediateExecuteArgs = append(immediateExecuteArgs, []string{ + crushDeviceClassFlag, + device.Config.DeviceClass, + }...) + } + // Reporting immediateReportArgs := append(immediateExecuteArgs, []string{ "--report", @@ -245,10 +261,37 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO } } - for md, devs := range metadataDevices { + for md, conf := range metadataDevices { - mdArgs := append(batchArgs, devs...) - mdArgs = append(mdArgs, path.Join("/dev", md)) + mdArgs := batchArgs + if _, ok := conf["osdsperdevice"]; ok { + mdArgs = append(mdArgs, []string{ + osdsPerDeviceFlag, + conf["osdsperdevice"], + }...) + } + if _, ok := conf["deviceclass"]; ok { + mdArgs = append(mdArgs, []string{ + crushDeviceClassFlag, + conf["deviceclass"], + }...) + } + if _, ok := conf["databasesizemb"]; ok { + mdArgs = append(mdArgs, []string{ + databaseSizeFlag, + conf["databasesizemb"], + }...) + } + mdArgs = append(mdArgs, strings.Split(conf["devices"], " ")...) + + if a.cluster.CephVersion.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 1}) { + mdArgs = append(mdArgs, []string{ + dbDeviceFlag, + path.Join("/dev", md), + }...) + } else { + mdArgs = append(mdArgs, path.Join("/dev", md)) + } // Reporting reportArgs := append(mdArgs, []string{ @@ -289,6 +332,13 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO return nil } +func getDatabaseSize(globalSize int, deviceSize int) int { + if deviceSize > 0 { + globalSize = deviceSize + } + return globalSize +} + func sanitizeOSDsPerDevice(count int) string { if count < 1 { count = 1 diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 5de99401e..ac08a2041 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -306,3 +306,8 @@ func TestSanitizeOSDsPerDevice(t *testing.T) { assert.Equal(t, "1", sanitizeOSDsPerDevice(1)) assert.Equal(t, "2", sanitizeOSDsPerDevice(2)) } + +func TestGetDatabaseSize(t *testing.T) { + assert.Equal(t, 0, getDatabaseSize(0, 0)) + assert.Equal(t, 2048, getDatabaseSize(4096, 2048)) +} diff --git a/pkg/operator/ceph/cluster/osd/config/config.go b/pkg/operator/ceph/cluster/osd/config/config.go index 5495a8360..47c1e20b3 100644 --- a/pkg/operator/ceph/cluster/osd/config/config.go +++ b/pkg/operator/ceph/cluster/osd/config/config.go @@ -44,6 +44,7 @@ const ( OSDsPerDeviceKey = "osdsPerDevice" EncryptedDeviceKey = "encryptedDevice" MetadataDeviceKey = "metadataDevice" + DeviceClassKey = "deviceClass" ) type StoreConfig struct { @@ -53,6 +54,7 @@ type StoreConfig struct { JournalSizeMB int `json:"journalSizeMB,omitempty"` OSDsPerDevice int `json:"osdsPerDevice,omitempty"` EncryptedDevice bool `json:"encryptedDevice,omitempty"` + DeviceClass string `json:"deviceClass,omitempty"` } func ToStoreConfig(config map[string]string) StoreConfig { @@ -71,6 +73,8 @@ func ToStoreConfig(config map[string]string) StoreConfig { storeConfig.OSDsPerDevice = convertToIntIgnoreErr(v) case EncryptedDeviceKey: storeConfig.EncryptedDevice = (v == "true") + case DeviceClassKey: + storeConfig.DeviceClass = v } } diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index bfc49472b..0fed9ad36 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -555,9 +555,23 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun } else { devSuffix += ":1" } + if databaseSizeMB, ok := device.Config[config.DatabaseSizeMBKey]; ok { + logger.Infof("osd %s requested with DB size %sMB (node %s)", device.Name, databaseSizeMB, osdProps.crushHostname) + devSuffix += ":" + databaseSizeMB + } else { + devSuffix += ":" + } + if deviceClass, ok := device.Config[config.DeviceClassKey]; ok { + logger.Infof("osd %s requested with deviceClass %s (node %s)", device.Name, deviceClass, osdProps.crushHostname) + devSuffix += ":" + deviceClass + } else { + devSuffix += ":" + } if md, ok := device.Config[config.MetadataDeviceKey]; ok { logger.Infof("osd %s requested with metadataDevice %s (node %s)", device.Name, md, osdProps.crushHostname) devSuffix += ":" + md + } else { + devSuffix += ":" } deviceNames[i] = device.Name + devSuffix } @@ -570,6 +584,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun envVars = append(envVars, deviceFilterEnvVar("all")) devMountNeeded = true } + envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()}) if osdProps.metadataDevice != "" { envVars = append(envVars, metadataDeviceEnvVar(osdProps.metadataDevice)) diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 69903b5a8..03a5fb4dd 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -38,7 +38,7 @@ import ( ) func TestPodContainer(t *testing.T) { - cluster := &Cluster{Namespace: "myosd", rookVersion: "23", cephVersion: cephv1.CephVersionSpec{}} + cluster := &Cluster{Namespace: "myosd", rookVersion: "23", cephVersion: cephv1.CephVersionSpec{}, clusterInfo: &cephconfig.ClusterInfo{}} osdProps := osdProperties{ crushHostname: "node", devices: []rookalpha.Device{},