ceph: add support for disk selection by full path
Rook-Ceph now supports specify device by their fullpath instead of links created by the kernel like /dev/sdb. Closes: https://github.com/rook/rook/issues/1228 Co-authored-by: Sean Micklethwaite <sean@wayve.ai> Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
1 parent
b7ba466b0b
commit
b04e35dab5
6 files changed
+30
-7
No files matched your search
@@ -250,7 +250,7 @@ Below are the settings available, both at the cluster and individual node level,
|
||||
* `^/dev/sd.`: Selects all devices starting with `sd`
|
||||
* `^/dev/disk/by-path/pci-.*`: Selects all devices which are connected to PCI bus
|
||||
* `devices`: A list of individual device names belonging to this node to include in the storage cluster.
|
||||
* `name`: The name of the device (e.g., `sda`)
|
||||
* `name`: The name of the device (e.g., `sda`), or full udev path (e.g. `/dev/disk/by-id/ata-ST4000DM004-XXXX` - this will not change after reboots).
|
||||
* `config`: Device-specific config settings. See the [config settings](#osd-configuration-settings) below
|
||||
* `storageClassDeviceSets`: Explained in [Storage Class Device Sets](#storage-class-device-sets)
|
||||
|
||||
@@ -442,7 +442,7 @@ spec:
|
||||
- name: "172.17.4.201"
|
||||
devices: # specific devices to use for storage can be specified for each node
|
||||
- name: "sdb"
|
||||
- name: "sdc"
|
||||
- name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # both device name and explicit udev links are supported
|
||||
config: # configuration can be specified at the node level which overrides the cluster level config
|
||||
storeType: bluestore
|
||||
- name: "172.17.4.301"
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
- OSD on PVC now supports PVC expansion, if the size of the underlying block increases the Bluestore main block and the overall storage capacity will grow up.
|
||||
- Ceph Nautilus 14.2.5 is the minimum supported version
|
||||
- OSD on PVC doesn't use LVM anymore to configure OSD, but solely relies on the entire block device, done [here](https://github.com/rook/rook/pull/4435).
|
||||
- Specific devices for OSDs can now be specified using the full udev path (e.g. /dev/disk/by-id/ata-ST4000DM004-XXXX) instead of the device name.
|
||||
|
||||
|
||||
### EdgeFS
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ spec:
|
||||
# - name: "nvme01" # multiple osds can be created on high performance devices
|
||||
# config:
|
||||
# osdsPerDevice: "5"
|
||||
# - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths
|
||||
# config: # configuration can be specified at the node level which overrides the cluster level config
|
||||
# storeType: filestore
|
||||
# - name: "172.17.4.301"
|
||||
|
||||
@@ -380,6 +380,15 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi
|
||||
} else if device.Name == desiredDevice.Name {
|
||||
logger.Infof("%q found in the desired devices", device.Name)
|
||||
matched = true
|
||||
} else if strings.HasPrefix(desiredDevice.Name, "/dev/") {
|
||||
devLinks := strings.Split(device.DevLinks, " ")
|
||||
for _, link := range devLinks {
|
||||
if link == desiredDevice.Name {
|
||||
logger.Infof("%q found in the desired devices (matched by link: %q)", device.Name, link)
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
matchedDevice = desiredDevice
|
||||
if matched {
|
||||
|
||||
@@ -181,6 +181,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
|
||||
{Name: "sdb", DevLinks: "/dev/disk/by-id/scsi-4567 /dev/disk/by-path/pci-4:5:6:7-scsi-1", RealName: "sdb"},
|
||||
{Name: "sdc", DevLinks: "/dev/disk/by-id/scsi-89ab /dev/disk/by-path/pci-8:9:a:b-scsi-1", RealName: "sdc"},
|
||||
{Name: "sdd", DevLinks: "/dev/disk/by-id/scsi-cdef /dev/disk/by-path/pci-c:d:e:f-scsi-1", RealName: "sdd"},
|
||||
{Name: "sde", DevLinks: "/dev/disk/by-id/sde-0x0000 /dev/disk/by-path/pci-0000:00:18.0-ata-1", RealName: "sde"},
|
||||
{Name: "nvme01", DevLinks: "/dev/disk/by-id/nvme-0246 /dev/disk/by-path/pci-0:2:4:6-nvme-1", RealName: "nvme01"},
|
||||
{Name: "rda", RealName: "rda"},
|
||||
{Name: "rdb", RealName: "rdb"},
|
||||
@@ -194,7 +195,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
|
||||
pvcBackedOSD := false
|
||||
mapping, err := getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD, version)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 6, len(mapping.Entries))
|
||||
assert.Equal(t, 7, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["rda"].Data)
|
||||
@@ -207,7 +208,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
|
||||
version = cephver.Nautilus
|
||||
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD, version)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 5, len(mapping.Entries))
|
||||
assert.Equal(t, 6, len(mapping.Entries))
|
||||
|
||||
// Do not skip partition anymore
|
||||
version = cephver.Octopus
|
||||
@@ -224,7 +225,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
|
||||
// select the sd* devices
|
||||
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^sd.$", IsFilter: true}}, "", pvcBackedOSD, version)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(mapping.Entries))
|
||||
assert.Equal(t, 3, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
|
||||
@@ -245,7 +246,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
|
||||
// select the sd* devices by path names
|
||||
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^/dev/sd.$", IsDevicePathFilter: true}}, "", pvcBackedOSD, version)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(mapping.Entries))
|
||||
assert.Equal(t, 3, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
|
||||
@@ -256,6 +257,12 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
|
||||
assert.Equal(t, -1, mapping.Entries["sda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
|
||||
// select a device by explicit link
|
||||
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "/dev/disk/by-id/sde-0x0000"}}, "", pvcBackedOSD, version)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sde"].Data)
|
||||
|
||||
// test on PVC
|
||||
pvcBackedOSD = true
|
||||
mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "", pvcBackedOSD, version)
|
||||
|
||||
@@ -871,7 +871,11 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
|
||||
} else {
|
||||
devSuffix += ":"
|
||||
}
|
||||
deviceNames[i] = device.Name + devSuffix
|
||||
deviceID := device.Name
|
||||
if device.FullPath != "" {
|
||||
deviceID = device.FullPath
|
||||
}
|
||||
deviceNames[i] = deviceID + devSuffix
|
||||
}
|
||||
envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ",")))
|
||||
devMountNeeded = true
|
||||
|
||||
Reference in new issue
Block a user