ceph: Add support for per OSD metdataDevices

Add ceph-volume support for per OSD metdataDevices

Signed-off-by: Michael Vollman <michael.b.vollman@gmail.com>
This commit is contained in:
Michael Vollman committed 2019-08-02 07:41:45 -04:00
1 parent 86cccaa554
commit f3ef680298
6 files changed
+60 -21

No files matched your search

+1
View File
@@ -11,6 +11,7 @@ an example usage
- Linear disk device can now be used for Ceph OSDs.
- The integration tests can be triggered for specific storage providers rather than always running all tests. See the [dev guide](INSTALL.md#test-storage-provider) for more details.
- Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph.
- Allow `metadataDevice` to be set per OSD device in the device specific `config` section.
### Ceph
+8 -2
View File
@@ -268,9 +268,12 @@ func commonOSDInit(cmd *cobra.Command) {
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
}
// Parse the devices, which are comma separated. A colon indicates a non-default number of osds per device.
// Parse the devices, which are comma separated. A colon indicates a non-default number of osds per device
// or a non collocated metadata device.
// For example, one osd will be created on each of sda and sdb, with 5 osds on the nvme01 device.
// sda,sdb,nvme01:5
// sda:1,sdb:1,nvme01:5
// For example, 3 osds will use sdb SSD for db and 3 osds will use sdc SSD for db.
// sdd:1:sdb,sde:1:sdb,sdf:1:sdb,sdg:1:sdc,sdh:1:sdc,sdi:1:sdc
func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) {
var result []osddaemon.DesiredDevice
parsed := strings.Split(devices, ",")
@@ -287,6 +290,9 @@ func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) {
}
d.OSDsPerDevice = count
}
if len(parts) > 2 {
d.MetadataDevice = parts[2]
}
result = append(result, d)
}
+21
View File
@@ -48,4 +48,25 @@ func TestParseDesiredDevices(t *testing.T) {
result, err = parseDevices(devices)
assert.Nil(t, result)
assert.NotNil(t, err)
// metadataDevice and OSDsPerDevice
devices = "sdd:1:sdb,sde:1:sdb,sdf:1:sdc,sdg:1:sdc"
result, err = parseDevices(devices)
assert.Equal(t, "sdd", result[0].Name)
assert.Equal(t, "sde", result[1].Name)
assert.Equal(t, "sdf", result[2].Name)
assert.Equal(t, "sdg", result[3].Name)
assert.Equal(t, 1, result[0].OSDsPerDevice)
assert.Equal(t, 1, result[1].OSDsPerDevice)
assert.Equal(t, 1, result[2].OSDsPerDevice)
assert.Equal(t, 1, result[3].OSDsPerDevice)
assert.Equal(t, "sdb", result[0].MetadataDevice)
assert.Equal(t, "sdb", result[1].MetadataDevice)
assert.Equal(t, "sdc", result[2].MetadataDevice)
assert.Equal(t, "sdc", result[3].MetadataDevice)
assert.False(t, result[0].IsFilter)
assert.False(t, result[1].IsFilter)
assert.False(t, result[2].IsFilter)
assert.False(t, result[3].IsFilter)
}
+4 -3
View File
@@ -73,9 +73,10 @@ type Device struct {
// DesiredDevice keeps track of the desired settings for a device
type DesiredDevice struct {
Name string
OSDsPerDevice int
IsFilter bool
Name string
OSDsPerDevice int
MetadataDevice string
IsFilter bool
}
type DeviceOsdMapping struct {
+18 -13
View File
@@ -98,15 +98,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
}
}
// When mixed hdd/ssd devices are given, ceph-volume configures db lv on the ssd.
metadataDeviceSpecified := false
if a.metadataDevice != "" {
logger.Infof("using %s as metadataDevice and let ceph-volume lvm batch decide how to create volumes", a.metadataDevice)
metadataDeviceSpecified = true
batchArgs = append(batchArgs, path.Join("/dev", a.metadataDevice))
}
configured := 0
metadataDevices := make(map[string][]string)
for name, device := range devices.Entries {
if device.LegacyPartitionsFound {
logger.Infof("skipping device %s configured with legacy rook osd", name)
@@ -116,10 +108,19 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
if device.Data == -1 {
logger.Infof("configuring new device %s", name)
deviceArg := path.Join("/dev", name)
if metadataDeviceSpecified {
if a.metadataDevice != "" || device.Config.MetadataDevice != "" {
// When mixed hdd/ssd devices are given, ceph-volume configures db lv on the ssd.
// the device will be configured as a batch at the end of the method
batchArgs = append(batchArgs, deviceArg)
configured++
md := a.metadataDevice
if device.Config.MetadataDevice != "" {
md = device.Config.MetadataDevice
}
logger.Infof("using %s as metadataDevice for device %s and let ceph-volume lvm batch decide how to create volumes", md, deviceArg)
if _, ok := metadataDevices[md]; ok {
metadataDevices[md] = append(metadataDevices[md], deviceArg)
} else {
metadataDevices[md] = []string{deviceArg}
}
} else {
immediateExecuteArgs := append(baseArgs, []string{
deviceArg,
@@ -147,7 +148,11 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
}
}
if configured > 0 {
for md, devs := range metadataDevices {
batchArgs = append(batchArgs, path.Join("/dev", md))
batchArgs = append(batchArgs, devs...)
// Reporting
reportArgs := append(batchArgs, []string{
"--report",
+8 -3
View File
@@ -492,12 +492,17 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro
if len(devices) > 0 {
deviceNames := make([]string, len(devices))
for i, device := range devices {
countSuffix := ""
devSuffix := ""
count := "1"
if count, ok := device.Config[config.OSDsPerDeviceKey]; ok {
logger.Infof("%s osds requested on device %s (node %s)", count, device.Name, nodeName)
countSuffix = ":" + count
}
deviceNames[i] = device.Name + countSuffix
devSuffix += ":" + count
if md, ok := device.Config[config.MetadataDeviceKey]; ok {
logger.Infof("osd %s requested with metadataDevice %s (node %s)", device.Name, md, nodeName)
devSuffix += ":" + md
}
deviceNames[i] = device.Name + devSuffix
}
envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ",")))
devMountNeeded = true