ceph: add support for disk selection by full path

Rook-Ceph now supports specify device by their fullpath instead of links
created by the kernel like /dev/sdb.

Closes: https://github.com/rook/rook/issues/1228
Co-authored-by: Sean Micklethwaite <sean@wayve.ai>
Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien HanandSean Micklethwaite committed 2020-02-13 17:31:38 +01:00
1 parent b7ba466b0b
commit b04e35dab5
6 files changed
+30 -7

No files matched your search

+2 -2
View File
@@ -250,7 +250,7 @@ Below are the settings available, both at the cluster and individual node level,
* `^/dev/sd.`: Selects all devices starting with `sd`
* `^/dev/disk/by-path/pci-.*`: Selects all devices which are connected to PCI bus
* `devices`: A list of individual device names belonging to this node to include in the storage cluster.
* `name`: The name of the device (e.g., `sda`)
* `name`: The name of the device (e.g., `sda`), or full udev path (e.g. `/dev/disk/by-id/ata-ST4000DM004-XXXX` - this will not change after reboots).
* `config`: Device-specific config settings. See the [config settings](#osd-configuration-settings) below
* `storageClassDeviceSets`: Explained in [Storage Class Device Sets](#storage-class-device-sets)
@@ -442,7 +442,7 @@ spec:
- name: "172.17.4.201"
devices: # specific devices to use for storage can be specified for each node
- name: "sdb"
- name: "sdc"
- name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # both device name and explicit udev links are supported
config: # configuration can be specified at the node level which overrides the cluster level config
storeType: bluestore
- name: "172.17.4.301"
+2
View File
@@ -13,6 +13,8 @@
- OSD on PVC now supports PVC expansion, if the size of the underlying block increases the Bluestore main block and the overall storage capacity will grow up.
- Ceph Nautilus 14.2.5 is the minimum supported version
- OSD on PVC doesn't use LVM anymore to configure OSD, but solely relies on the entire block device, done [here](https://github.com/rook/rook/pull/4435).
- Specific devices for OSDs can now be specified using the full udev path (e.g. /dev/disk/by-id/ata-ST4000DM004-XXXX) instead of the device name.
### EdgeFS
@@ -149,6 +149,7 @@ spec:
# - name: "nvme01" # multiple osds can be created on high performance devices
# config:
# osdsPerDevice: "5"
# - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths
# config: # configuration can be specified at the node level which overrides the cluster level config
# storeType: filestore
# - name: "172.17.4.301"
+9
View File
@@ -380,6 +380,15 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi
} else if device.Name == desiredDevice.Name {
logger.Infof("%q found in the desired devices", device.Name)
matched = true
} else if strings.HasPrefix(desiredDevice.Name, "/dev/") {
devLinks := strings.Split(device.DevLinks, " ")
for _, link := range devLinks {
if link == desiredDevice.Name {
logger.Infof("%q found in the desired devices (matched by link: %q)", device.Name, link)
matched = true
break
}
}
}
matchedDevice = desiredDevice
if matched {
+11 -4
View File
@@ -181,6 +181,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
{Name: "sdb", DevLinks: "/dev/disk/by-id/scsi-4567 /dev/disk/by-path/pci-4:5:6:7-scsi-1", RealName: "sdb"},
{Name: "sdc", DevLinks: "/dev/disk/by-id/scsi-89ab /dev/disk/by-path/pci-8:9:a:b-scsi-1", RealName: "sdc"},
{Name: "sdd", DevLinks: "/dev/disk/by-id/scsi-cdef /dev/disk/by-path/pci-c:d:e:f-scsi-1", RealName: "sdd"},
{Name: "sde", DevLinks: "/dev/disk/by-id/sde-0x0000 /dev/disk/by-path/pci-0000:00:18.0-ata-1", RealName: "sde"},
{Name: "nvme01", DevLinks: "/dev/disk/by-id/nvme-0246 /dev/disk/by-path/pci-0:2:4:6-nvme-1", RealName: "nvme01"},
{Name: "rda", RealName: "rda"},
{Name: "rdb", RealName: "rdb"},
@@ -194,7 +195,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
pvcBackedOSD := false
mapping, err := getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD, version)
assert.Nil(t, err)
assert.Equal(t, 6, len(mapping.Entries))
assert.Equal(t, 7, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sda"].Data)
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
assert.Equal(t, -1, mapping.Entries["rda"].Data)
@@ -207,7 +208,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
version = cephver.Nautilus
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD, version)
assert.Nil(t, err)
assert.Equal(t, 5, len(mapping.Entries))
assert.Equal(t, 6, len(mapping.Entries))
// Do not skip partition anymore
version = cephver.Octopus
@@ -224,7 +225,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
// select the sd* devices
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^sd.$", IsFilter: true}}, "", pvcBackedOSD, version)
assert.Nil(t, err)
assert.Equal(t, 2, len(mapping.Entries))
assert.Equal(t, 3, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sda"].Data)
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
@@ -245,7 +246,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
// select the sd* devices by path names
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^/dev/sd.$", IsDevicePathFilter: true}}, "", pvcBackedOSD, version)
assert.Nil(t, err)
assert.Equal(t, 2, len(mapping.Entries))
assert.Equal(t, 3, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sda"].Data)
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
@@ -256,6 +257,12 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
assert.Equal(t, -1, mapping.Entries["sda"].Data)
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
// select a device by explicit link
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "/dev/disk/by-id/sde-0x0000"}}, "", pvcBackedOSD, version)
assert.Nil(t, err)
assert.Equal(t, 1, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sde"].Data)
// test on PVC
pvcBackedOSD = true
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "", pvcBackedOSD, version)
+5 -1
View File
@@ -871,7 +871,11 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
} else {
devSuffix += ":"
}
deviceNames[i] = device.Name + devSuffix
deviceID := device.Name
if device.FullPath != "" {
deviceID = device.FullPath
}
deviceNames[i] = deviceID + devSuffix
}
envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ",")))
devMountNeeded = true