ceph: Add support for per OSD metdataDevices
Add ceph-volume support for per OSD metdataDevices Signed-off-by: Michael Vollman <michael.b.vollman@gmail.com>
This commit is contained in:
1 parent
86cccaa554
commit
f3ef680298
6 files changed
+60
-21
No files matched your search
@@ -11,6 +11,7 @@ an example usage
|
||||
- Linear disk device can now be used for Ceph OSDs.
|
||||
- The integration tests can be triggered for specific storage providers rather than always running all tests. See the [dev guide](INSTALL.md#test-storage-provider) for more details.
|
||||
- Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph.
|
||||
- Allow `metadataDevice` to be set per OSD device in the device specific `config` section.
|
||||
|
||||
### Ceph
|
||||
|
||||
|
||||
@@ -268,9 +268,12 @@ func commonOSDInit(cmd *cobra.Command) {
|
||||
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
|
||||
}
|
||||
|
||||
// Parse the devices, which are comma separated. A colon indicates a non-default number of osds per device.
|
||||
// Parse the devices, which are comma separated. A colon indicates a non-default number of osds per device
|
||||
// or a non collocated metadata device.
|
||||
// For example, one osd will be created on each of sda and sdb, with 5 osds on the nvme01 device.
|
||||
// sda,sdb,nvme01:5
|
||||
// sda:1,sdb:1,nvme01:5
|
||||
// For example, 3 osds will use sdb SSD for db and 3 osds will use sdc SSD for db.
|
||||
// sdd:1:sdb,sde:1:sdb,sdf:1:sdb,sdg:1:sdc,sdh:1:sdc,sdi:1:sdc
|
||||
func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) {
|
||||
var result []osddaemon.DesiredDevice
|
||||
parsed := strings.Split(devices, ",")
|
||||
@@ -287,6 +290,9 @@ func parseDevices(devices string) ([]osddaemon.DesiredDevice, error) {
|
||||
}
|
||||
d.OSDsPerDevice = count
|
||||
}
|
||||
if len(parts) > 2 {
|
||||
d.MetadataDevice = parts[2]
|
||||
}
|
||||
result = append(result, d)
|
||||
}
|
||||
|
||||
|
||||
@@ -48,4 +48,25 @@ func TestParseDesiredDevices(t *testing.T) {
|
||||
result, err = parseDevices(devices)
|
||||
assert.Nil(t, result)
|
||||
assert.NotNil(t, err)
|
||||
|
||||
// metadataDevice and OSDsPerDevice
|
||||
devices = "sdd:1:sdb,sde:1:sdb,sdf:1:sdc,sdg:1:sdc"
|
||||
result, err = parseDevices(devices)
|
||||
assert.Equal(t, "sdd", result[0].Name)
|
||||
assert.Equal(t, "sde", result[1].Name)
|
||||
assert.Equal(t, "sdf", result[2].Name)
|
||||
assert.Equal(t, "sdg", result[3].Name)
|
||||
assert.Equal(t, 1, result[0].OSDsPerDevice)
|
||||
assert.Equal(t, 1, result[1].OSDsPerDevice)
|
||||
assert.Equal(t, 1, result[2].OSDsPerDevice)
|
||||
assert.Equal(t, 1, result[3].OSDsPerDevice)
|
||||
assert.Equal(t, "sdb", result[0].MetadataDevice)
|
||||
assert.Equal(t, "sdb", result[1].MetadataDevice)
|
||||
assert.Equal(t, "sdc", result[2].MetadataDevice)
|
||||
assert.Equal(t, "sdc", result[3].MetadataDevice)
|
||||
assert.False(t, result[0].IsFilter)
|
||||
assert.False(t, result[1].IsFilter)
|
||||
assert.False(t, result[2].IsFilter)
|
||||
assert.False(t, result[3].IsFilter)
|
||||
|
||||
}
|
||||
@@ -73,9 +73,10 @@ type Device struct {
|
||||
|
||||
// DesiredDevice keeps track of the desired settings for a device
|
||||
type DesiredDevice struct {
|
||||
Name string
|
||||
OSDsPerDevice int
|
||||
IsFilter bool
|
||||
Name string
|
||||
OSDsPerDevice int
|
||||
MetadataDevice string
|
||||
IsFilter bool
|
||||
}
|
||||
|
||||
type DeviceOsdMapping struct {
|
||||
|
||||
@@ -98,15 +98,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
|
||||
}
|
||||
}
|
||||
|
||||
// When mixed hdd/ssd devices are given, ceph-volume configures db lv on the ssd.
|
||||
metadataDeviceSpecified := false
|
||||
if a.metadataDevice != "" {
|
||||
logger.Infof("using %s as metadataDevice and let ceph-volume lvm batch decide how to create volumes", a.metadataDevice)
|
||||
metadataDeviceSpecified = true
|
||||
batchArgs = append(batchArgs, path.Join("/dev", a.metadataDevice))
|
||||
}
|
||||
|
||||
configured := 0
|
||||
metadataDevices := make(map[string][]string)
|
||||
for name, device := range devices.Entries {
|
||||
if device.LegacyPartitionsFound {
|
||||
logger.Infof("skipping device %s configured with legacy rook osd", name)
|
||||
@@ -116,10 +108,19 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
|
||||
if device.Data == -1 {
|
||||
logger.Infof("configuring new device %s", name)
|
||||
deviceArg := path.Join("/dev", name)
|
||||
if metadataDeviceSpecified {
|
||||
if a.metadataDevice != "" || device.Config.MetadataDevice != "" {
|
||||
// When mixed hdd/ssd devices are given, ceph-volume configures db lv on the ssd.
|
||||
// the device will be configured as a batch at the end of the method
|
||||
batchArgs = append(batchArgs, deviceArg)
|
||||
configured++
|
||||
md := a.metadataDevice
|
||||
if device.Config.MetadataDevice != "" {
|
||||
md = device.Config.MetadataDevice
|
||||
}
|
||||
logger.Infof("using %s as metadataDevice for device %s and let ceph-volume lvm batch decide how to create volumes", md, deviceArg)
|
||||
if _, ok := metadataDevices[md]; ok {
|
||||
metadataDevices[md] = append(metadataDevices[md], deviceArg)
|
||||
} else {
|
||||
metadataDevices[md] = []string{deviceArg}
|
||||
}
|
||||
} else {
|
||||
immediateExecuteArgs := append(baseArgs, []string{
|
||||
deviceArg,
|
||||
@@ -147,7 +148,11 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
|
||||
}
|
||||
}
|
||||
|
||||
if configured > 0 {
|
||||
for md, devs := range metadataDevices {
|
||||
|
||||
batchArgs = append(batchArgs, path.Join("/dev", md))
|
||||
batchArgs = append(batchArgs, devs...)
|
||||
|
||||
// Reporting
|
||||
reportArgs := append(batchArgs, []string{
|
||||
"--report",
|
||||
|
||||
@@ -492,12 +492,17 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro
|
||||
if len(devices) > 0 {
|
||||
deviceNames := make([]string, len(devices))
|
||||
for i, device := range devices {
|
||||
countSuffix := ""
|
||||
devSuffix := ""
|
||||
count := "1"
|
||||
if count, ok := device.Config[config.OSDsPerDeviceKey]; ok {
|
||||
logger.Infof("%s osds requested on device %s (node %s)", count, device.Name, nodeName)
|
||||
countSuffix = ":" + count
|
||||
}
|
||||
deviceNames[i] = device.Name + countSuffix
|
||||
devSuffix += ":" + count
|
||||
if md, ok := device.Config[config.MetadataDeviceKey]; ok {
|
||||
logger.Infof("osd %s requested with metadataDevice %s (node %s)", device.Name, md, nodeName)
|
||||
devSuffix += ":" + md
|
||||
}
|
||||
deviceNames[i] = device.Name + devSuffix
|
||||
}
|
||||
envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ",")))
|
||||
devMountNeeded = true
|
||||
|
||||
Reference in new issue
Block a user