diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index c5b1675ba..f85d5daba 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -229,7 +229,7 @@ This feature is only available when `useAllNodes` has been set to `false`. Below are the settings available, both at the cluster and individual node level, for selecting which storage resources will be included in the cluster. -* `useAllDevices`: `true` or `false`, indicating whether all devices found on nodes in the cluster should be automatically consumed by OSDs. **Not recommended** unless you have a very controlled environment where you will not risk formatting of devices with existing data. When `true`, all devices will be used except those with partitions created or a local filesystem. Is overridden by `deviceFilter` if specified. +* `useAllDevices`: `true` or `false`, indicating whether all devices found on nodes in the cluster should be automatically consumed by OSDs. **Not recommended** unless you have a very controlled environment where you will not risk formatting of devices with existing data. When `true`, all devices/partitions will be used. Is overridden by `deviceFilter` if specified. * `deviceFilter`: A regular expression for short kernel names of devices (e.g. `sda`) that allows selection of devices to be consumed by OSDs. If individual devices have been specified for a node then this filter will be ignored. This field uses [golang regular expression syntax](https://golang.org/pkg/regexp/syntax/). For example: * `sdb`: Only selects the `sdb` device if found * `^sd.`: Selects all devices starting with `sd` diff --git a/pkg/clusterd/disk.go b/pkg/clusterd/disk.go index 80fe328b5..4941b2bf0 100644 --- a/pkg/clusterd/disk.go +++ b/pkg/clusterd/disk.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package clusterd import ( @@ -32,7 +33,7 @@ var ( isRBD = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$") ) -// check whether a device is completely empty +// GetDeviceEmpty check whether a device is completely empty func GetDeviceEmpty(device *sys.LocalDisk) bool { return device.Parent == "" && (device.Type == sys.DiskType || device.Type == sys.SSDType || device.Type == sys.CryptType || device.Type == sys.LVMType) && len(device.Partitions) == 0 && device.Filesystem == "" } @@ -41,7 +42,7 @@ func ignoreDevice(d string) bool { return isRBD.MatchString(d) } -// Discover all the details of devices available on the local node +// DiscoverDevices all the details of devices available on the local node func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) { var disks []*sys.LocalDisk devices, err := sys.ListDevices(executor) @@ -66,6 +67,24 @@ func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) { // go on without udev info logger.Warningf("failed to get udev info for device %s: %+v", d, err) } + + // Test if device has child, if so we skip it and only consider the partitions + // which will come in later iterations of the loop + // We only test if the type is 'disk', this is a property reported by lsblk + // and means it's a parent block device + if disk.Type == sys.DiskType { + deviceChild, err := sys.ListDevicesChild(executor, d) + if err != nil { + logger.Warningf("failed to detect child devices for device %q, assuming they are none. %+v", d, err) + } + // lsblk will output at least 2 lines if they are partitions, one for the parent + // and N for the child + if len(deviceChild) > 1 { + logger.Infof("skipping device %q because it has child, considering the child instead.", d) + continue + } + } + disks = append(disks, disk) } diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 058ecdabd..b7ac84247 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -283,17 +283,25 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi available := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}} for _, device := range context.Devices { - if device.Type == sys.PartType { - continue - } - partCount, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name, pvcBacked) - if err != nil { - return nil, errors.Wrapf(err, "failed to get device %q info", device.Name) + var partCount int + var ownPartitions bool + var err error + var fs string + if device.Type != sys.PartType { + partCount, ownPartitions, fs, err = sys.CheckIfDeviceAvailable(context.Executor, device.Name, pvcBacked) + if err != nil { + return nil, errors.Wrapf(err, "failed to get device %q info", device.Name) + } + + if fs != "" || !ownPartitions { + // not OK to use the device because it has a filesystem or rook doesn't own all its partitions + logger.Infof("skipping device %q that is in use (not by rook). fs: %s, ownPartitions: %t", device.Name, fs, ownPartitions) + continue + } } - if fs != "" || !ownPartitions { - // not OK to use the device because it has a filesystem or rook doesn't own all its partitions - logger.Infof("skipping device %q that is in use (not by rook). fs: %s, ownPartitions: %t", device.Name, fs, ownPartitions) + if device.Filesystem != "" { + logger.Infof("skipping device %q because it contains a filesystem %q", device.Name, device.Filesystem) continue } @@ -306,7 +314,6 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID} } else if len(desiredDevices) > 0 { var matched bool - var err error var matchedDevice DesiredDevice for _, desiredDevice := range desiredDevices { if desiredDevice.IsFilter { diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go index 5605aaefc..7c88064ab 100644 --- a/pkg/util/sys/device.go +++ b/pkg/util/sys/device.go @@ -18,6 +18,7 @@ package sys import ( "fmt" "os" + "path" "strconv" "strings" @@ -481,3 +482,15 @@ func parseUdevInfo(output string) map[string]string { } return result } + +// ListDevicesChild list all child available on a device +func ListDevicesChild(executor exec.Executor, device string) ([]string, error) { + cmd := "lsblk for child" + + childListRaw, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk", "--noheadings", "--pairs", path.Join("/dev", device)) + if err != nil { + return []string{}, fmt.Errorf("failed to list child devices of %q. %+v", device, err) + } + + return strings.Split(childListRaw, "\n"), nil +} diff --git a/pkg/util/sys/device_test.go b/pkg/util/sys/device_test.go index c9673febc..5f36343c2 100644 --- a/pkg/util/sys/device_test.go +++ b/pkg/util/sys/device_test.go @@ -86,6 +86,12 @@ SUBSYSTEM=block ` ) +var ( + lsblkChildOutput = `NAME="ceph--cec981b8--2eca--45cd--bf91--a4472779f2a9-osd--data--428984b7--f94d--40cd--9cb7--1458e1613eab" MAJ:MIN="252:0" RM="0" SIZE="29G" RO="0" TYPE="lvm" MOUNTPOINT="" +NAME="vdb" MAJ:MIN="253:16" RM="0" SIZE="30G" RO="0" TYPE="disk" MOUNTPOINT="" +NAME="vdb1" MAJ:MIN="253:17" RM="0" SIZE="30G" RO="0" TYPE="part" MOUNTPOINT=""` +) + func TestFindUUID(t *testing.T) { output := `Disk /dev/sdb: 10485760 sectors, 5.0 GiB Logical sector size: 512 bytes @@ -209,3 +215,17 @@ func TestParseUdevInfo(t *testing.T) { m := parseUdevInfo(udevOutput) assert.Equal(t, m["ID_FS_TYPE"], "ext2") } + +func TestListDevicesChildListDevicesChild(t *testing.T) { + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, arg ...string) (string, error) { + logger.Infof("action %s command %s", actionName, command) + return lsblkChildOutput, nil + }, + } + + device := "/dev/vdb" + child, err := ListDevicesChild(executor, device) + assert.NoError(t, err) + assert.Equal(t, 3, len(child)) +}