From f3ef680298c8b3da6aa09a52f028ee492cbccf0c Mon Sep 17 00:00:00 2001 From: Michael Vollman Date: Mon, 20 May 2019 10:12:21 -0400 Subject: [PATCH] ceph: Add support for per OSD metdataDevices Add ceph-volume support for per OSD metdataDevices Signed-off-by: Michael Vollman --- PendingReleaseNotes.md | 1 + cmd/rook/ceph/osd.go | 10 +++++++-- cmd/rook/ceph/osd_test.go | 21 ++++++++++++++++++ pkg/daemon/ceph/osd/device.go | 7 +++--- pkg/daemon/ceph/osd/volume.go | 31 ++++++++++++++++----------- pkg/operator/ceph/cluster/osd/spec.go | 11 +++++++--- 6 files changed, 60 insertions(+), 21 deletions(-) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index c711a182f..88e9072fc 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -11,6 +11,7 @@ an example usage - Linear disk device can now be used for Ceph OSDs. - The integration tests can be triggered for specific storage providers rather than always running all tests. See the [dev guide](INSTALL.md#test-storage-provider) for more details. - Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph. +- Allow `metadataDevice` to be set per OSD device in the device specific `config` section. ### Ceph diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index ef2d25001..ee22f2106 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -268,9 +268,12 @@ func commonOSDInit(cmd *cobra.Command) { clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) } -// Parse the devices, which are comma separated. A colon indicates a non-default number of osds per device. +// Parse the devices, which are comma separated. A colon indicates a non-default number of osds per device +// or a non collocated metadata device. // For example, one osd will be created on each of sda and sdb, with 5 osds on the nvme01 device. -// sda,sdb,nvme01:5 +// sda:1,sdb:1,nvme01:5 +// For example, 3 osds will use sdb SSD for db and 3 osds will use sdc SSD for db. +// sdd:1:sdb,sde:1:sdb,sdf:1:sdb,sdg:1:sdc,sdh:1:sdc,sdi:1:sdc func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) { var result []osddaemon.DesiredDevice parsed := strings.Split(devices, ",") @@ -287,6 +290,9 @@ func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) { } d.OSDsPerDevice = count } + if len(parts) > 2 { + d.MetadataDevice = parts[2] + } result = append(result, d) } diff --git a/cmd/rook/ceph/osd_test.go b/cmd/rook/ceph/osd_test.go index d664b25e5..3188a1748 100644 --- a/cmd/rook/ceph/osd_test.go +++ b/cmd/rook/ceph/osd_test.go @@ -48,4 +48,25 @@ func TestParseDesiredDevices(t *testing.T) { result, err = parseDevices(devices) assert.Nil(t, result) assert.NotNil(t, err) + + // metadataDevice and OSDsPerDevice + devices = "sdd:1:sdb,sde:1:sdb,sdf:1:sdc,sdg:1:sdc" + result, err = parseDevices(devices) + assert.Equal(t, "sdd", result[0].Name) + assert.Equal(t, "sde", result[1].Name) + assert.Equal(t, "sdf", result[2].Name) + assert.Equal(t, "sdg", result[3].Name) + assert.Equal(t, 1, result[0].OSDsPerDevice) + assert.Equal(t, 1, result[1].OSDsPerDevice) + assert.Equal(t, 1, result[2].OSDsPerDevice) + assert.Equal(t, 1, result[3].OSDsPerDevice) + assert.Equal(t, "sdb", result[0].MetadataDevice) + assert.Equal(t, "sdb", result[1].MetadataDevice) + assert.Equal(t, "sdc", result[2].MetadataDevice) + assert.Equal(t, "sdc", result[3].MetadataDevice) + assert.False(t, result[0].IsFilter) + assert.False(t, result[1].IsFilter) + assert.False(t, result[2].IsFilter) + assert.False(t, result[3].IsFilter) + } diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index 70f9d8f8c..63639dc75 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -73,9 +73,10 @@ type Device struct { // DesiredDevice keeps track of the desired settings for a device type DesiredDevice struct { - Name string - OSDsPerDevice int - IsFilter bool + Name string + OSDsPerDevice int + MetadataDevice string + IsFilter bool } type DeviceOsdMapping struct { diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 5f5362924..1a1fa0d23 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -98,15 +98,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO } } - // When mixed hdd/ssd devices are given, ceph-volume configures db lv on the ssd. - metadataDeviceSpecified := false - if a.metadataDevice != "" { - logger.Infof("using %s as metadataDevice and let ceph-volume lvm batch decide how to create volumes", a.metadataDevice) - metadataDeviceSpecified = true - batchArgs = append(batchArgs, path.Join("/dev", a.metadataDevice)) - } - - configured := 0 + metadataDevices := make(map[string][]string) for name, device := range devices.Entries { if device.LegacyPartitionsFound { logger.Infof("skipping device %s configured with legacy rook osd", name) @@ -116,10 +108,19 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO if device.Data == -1 { logger.Infof("configuring new device %s", name) deviceArg := path.Join("/dev", name) - if metadataDeviceSpecified { + if a.metadataDevice != "" || device.Config.MetadataDevice != "" { + // When mixed hdd/ssd devices are given, ceph-volume configures db lv on the ssd. // the device will be configured as a batch at the end of the method - batchArgs = append(batchArgs, deviceArg) - configured++ + md := a.metadataDevice + if device.Config.MetadataDevice != "" { + md = device.Config.MetadataDevice + } + logger.Infof("using %s as metadataDevice for device %s and let ceph-volume lvm batch decide how to create volumes", md, deviceArg) + if _, ok := metadataDevices[md]; ok { + metadataDevices[md] = append(metadataDevices[md], deviceArg) + } else { + metadataDevices[md] = []string{deviceArg} + } } else { immediateExecuteArgs := append(baseArgs, []string{ deviceArg, @@ -147,7 +148,11 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO } } - if configured > 0 { + for md, devs := range metadataDevices { + + batchArgs = append(batchArgs, path.Join("/dev", md)) + batchArgs = append(batchArgs, devs...) + // Reporting reportArgs := append(batchArgs, []string{ "--report", diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index d8b368cc8..109f28e45 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -492,12 +492,17 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro if len(devices) > 0 { deviceNames := make([]string, len(devices)) for i, device := range devices { - countSuffix := "" + devSuffix := "" + count := "1" if count, ok := device.Config[config.OSDsPerDeviceKey]; ok { logger.Infof("%s osds requested on device %s (node %s)", count, device.Name, nodeName) - countSuffix = ":" + count } - deviceNames[i] = device.Name + countSuffix + devSuffix += ":" + count + if md, ok := device.Config[config.MetadataDeviceKey]; ok { + logger.Infof("osd %s requested with metadataDevice %s (node %s)", device.Name, md, nodeName) + devSuffix += ":" + md + } + deviceNames[i] = device.Name + devSuffix } envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ","))) devMountNeeded = true