From df3a8564b697a1156c0bc4961c3918d0a1aa484b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 30 Jan 2020 17:46:50 +0100 Subject: [PATCH] ceph: skip partitions depending on the ceph version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we detect a partition we should not use it if the ceph version is not at least 14.2.8 since it has the necessary changes to support partitions with ceph-volume. Signed-off-by: Sébastien Han --- pkg/daemon/ceph/osd/daemon.go | 14 ++++++++++-- pkg/daemon/ceph/osd/daemon_test.go | 35 ++++++++++++++++++++---------- pkg/daemon/ceph/osd/volume.go | 2 +- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 723b269da..77f689720 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/util/sys" ) @@ -194,7 +195,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) logger.Info("creating and starting the osds") // determine the set of devices that can/should be used for OSDs. - devices, err := getAvailableDevices(context, agent.devices, agent.metadataDevice, agent.pvcBacked) + devices, err := getAvailableDevices(context, agent.devices, agent.metadataDevice, agent.pvcBacked, agent.cluster.CephVersion) if err != nil { return errors.Wrap(err, "failed to get available devices") } @@ -257,7 +258,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) return nil } -func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevice, metadataDevice string, pvcBacked bool) (*DeviceOsdMapping, error) { +func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevice, metadataDevice string, pvcBacked bool, cephVersion cephver.CephVersion) (*DeviceOsdMapping, error) { available := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}} for _, device := range context.Devices { @@ -284,6 +285,15 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi deviceToCheck = device.RealName } + // If we detect a partition we have to make sure that ceph-volume will be able to consume it + // ceph-volume version 14.2.8 has the right code to support partitions + if device.Type == sys.PartType { + if !cephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) { + logger.Infof("skipping device %q because it is a partition and ceph version is too old, you need at least ceph %q", device.Name, cephVolumeRawModeMinCephVersion.String()) + continue + } + } + // Check if the desired device is available // When running on PVC we use the real device name instead of the Kubernetes mountpoint // Otherwise ceph-volume inventory will fail on the udevadm check diff --git a/pkg/daemon/ceph/osd/daemon_test.go b/pkg/daemon/ceph/osd/daemon_test.go index 345858dca..c652cf0dd 100644 --- a/pkg/daemon/ceph/osd/daemon_test.go +++ b/pkg/daemon/ceph/osd/daemon_test.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/rook/rook/pkg/util/sys" "github.com/stretchr/testify/assert" @@ -184,13 +185,16 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil {Name: "rda", RealName: "rda"}, {Name: "rdb", RealName: "rdb"}, {Name: "/mnt/set1-0-data-qfhfk", RealName: "xvdcy"}, + {Name: "sdt1", RealName: "sdt1", Type: sys.PartType}, } + version := cephver.Octopus + // select all devices, including nvme01 for metadata pvcBackedOSD := false - mapping, err := getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD) + mapping, err := getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD, version) assert.Nil(t, err) - assert.Equal(t, 5, len(mapping.Entries)) + assert.Equal(t, 6, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) assert.Equal(t, -1, mapping.Entries["rda"].Data) @@ -199,30 +203,39 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil assert.NotNil(t, mapping.Entries["nvme01"].Metadata) assert.Equal(t, 0, len(mapping.Entries["nvme01"].Metadata)) + // Partition is skipped + version = cephver.Nautilus + mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD, version) + assert.Nil(t, err) + assert.Equal(t, 5, len(mapping.Entries)) + + // Do not skip partition anymore + version = cephver.Octopus + // select no devices both using and not using a filter - mapping, err = getAvailableDevices(context, nil, "", pvcBackedOSD) + mapping, err = getAvailableDevices(context, nil, "", pvcBackedOSD, version) assert.Nil(t, err) assert.Equal(t, 0, len(mapping.Entries)) - mapping, err = getAvailableDevices(context, nil, "", pvcBackedOSD) + mapping, err = getAvailableDevices(context, nil, "", pvcBackedOSD, version) assert.Nil(t, err) assert.Equal(t, 0, len(mapping.Entries)) // select the sd* devices - mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^sd.$", IsFilter: true}}, "", pvcBackedOSD) + mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^sd.$", IsFilter: true}}, "", pvcBackedOSD, version) assert.Nil(t, err) assert.Equal(t, 2, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) // select an exact device - mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "sdd"}}, "", pvcBackedOSD) + mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "sdd"}}, "", pvcBackedOSD, version) assert.Nil(t, err) assert.Equal(t, 1, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sdd"].Data) // select all devices except those that have a prefix of "s" - mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^[^s]", IsFilter: true}}, "", pvcBackedOSD) + mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^[^s]", IsFilter: true}}, "", pvcBackedOSD, version) assert.Nil(t, err) assert.Equal(t, 3, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["rda"].Data) @@ -230,14 +243,14 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil assert.Equal(t, -1, mapping.Entries["nvme01"].Data) // select the sd* devices by path names - mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^/dev/sd.$", IsDevicePathFilter: true}}, "", pvcBackedOSD) + mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^/dev/sd.$", IsDevicePathFilter: true}}, "", pvcBackedOSD, version) assert.Nil(t, err) assert.Equal(t, 2, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) // select the SCSI devices - mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^/dev/disk/by-path/.*-scsi-.*", IsDevicePathFilter: true}}, "", pvcBackedOSD) + mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^/dev/disk/by-path/.*-scsi-.*", IsDevicePathFilter: true}}, "", pvcBackedOSD, version) assert.Nil(t, err) assert.Equal(t, 2, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) @@ -245,9 +258,9 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil // test on PVC pvcBackedOSD = true - mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "", pvcBackedOSD) + mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "", pvcBackedOSD, version) assert.Nil(t, err) - assert.Equal(t, 6, len(mapping.Entries)) + assert.Equal(t, 7, len(mapping.Entries)) } func TestGetVolumeGroupName(t *testing.T) { diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index b82bd41e1..4a13304ce 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -50,7 +50,7 @@ var ( cephLogDir = "/var/log/ceph" lvmConfPath = "/etc/lvm/lvm.conf" cvLogDir = "" - // The "ceph-volume raw" command is available since Ceph 14.2.8 + // The "ceph-volume raw" command is available since Ceph 14.2.8 as well as partition support in ceph-volume cephVolumeRawModeMinCephVersion = cephver.CephVersion{Major: 14, Minor: 2, Extra: 8} )