From 2359ca5249dcd07a010cb54fedfe7f4127f25b0c Mon Sep 17 00:00:00 2001 From: Jared Watts Date: Mon, 26 Jun 2017 17:54:04 -0700 Subject: [PATCH] osd: include metadata devices when launching OSD daemon in Kubernetes --- pkg/ceph/osd/daemon.go | 61 +++++++++++++++++++++---------------- pkg/ceph/osd/daemon_test.go | 28 ++++++++++------- pkg/ceph/osd/device.go | 8 +++++ pkg/ceph/osd/device_test.go | 41 ++++++++++++++++++++++++- 4 files changed, 100 insertions(+), 38 deletions(-) diff --git a/pkg/ceph/osd/daemon.go b/pkg/ceph/osd/daemon.go index 0269b23ea..f047f67e2 100644 --- a/pkg/ceph/osd/daemon.go +++ b/pkg/ceph/osd/daemon.go @@ -53,7 +53,7 @@ func Run(context *clusterd.Context, agent *OsdAgent) error { logger.Infof("creating and starting the osds") // initialize the desired osds - devices, err := getAvailableDevices(context, hardware.Disks, agent.devices, agent.usingDeviceFilter) + devices, err := getAvailableDevices(context, hardware.Disks, agent.devices, agent.metadataDevice, agent.usingDeviceFilter) if err != nil { return fmt.Errorf("failed to get available devices. %+v", err) } @@ -87,7 +87,7 @@ func Run(context *clusterd.Context, agent *OsdAgent) error { } func getAvailableDevices(context *clusterd.Context, devices []*inventory.LocalDisk, desiredDevices string, - usingDeviceFilter bool) (*DeviceOsdMapping, error) { + metadataDevice string, usingDeviceFilter bool) (*DeviceOsdMapping, error) { var deviceList []string if !usingDeviceFilter { @@ -103,33 +103,42 @@ func getAvailableDevices(context *clusterd.Context, devices []*inventory.LocalDi if err != nil { return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err) } - if fs == "" && ownPartitions { - if desiredDevices == "all" { - available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} - } else if desiredDevices != "" { - var matched bool - var err error - if usingDeviceFilter { - // the desired devices is a regular expression - matched, err = regexp.Match(desiredDevices, []byte(device.Name)) - } else { - for i := range deviceList { - if device.Name == deviceList[i] { - matched = true - break - } + + if fs != "" || !ownPartitions { + // not OK to use the device because it has a filesystem or rook doesn't own all its partitions + logger.Infof("skipping device %s that is in use (not by rook). fs: %s, ownPartitions: %t", device.Name, fs, ownPartitions) + continue + } + + if metadataDevice != "" && metadataDevice == device.Name { + // current device is desired as the metadata device + available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}} + } else if desiredDevices == "all" { + // user has specified all devices, use the current one for data + available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} + } else if desiredDevices != "" { + var matched bool + var err error + if usingDeviceFilter { + // the desired devices is a regular expression + matched, err = regexp.Match(desiredDevices, []byte(device.Name)) + } else { + for i := range deviceList { + if device.Name == deviceList[i] { + matched = true + break } } - - if err == nil && matched { - available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} - } else { - logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err) - } - - } else { - logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name) } + + if err == nil && matched { + // the current device matches the user specifies filter/list, use it for data + available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} + } else { + logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err) + } + } else { + logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name) } } diff --git a/pkg/ceph/osd/daemon_test.go b/pkg/ceph/osd/daemon_test.go index 4c81b7d8e..d920792f8 100644 --- a/pkg/ceph/osd/daemon_test.go +++ b/pkg/ceph/osd/daemon_test.go @@ -94,7 +94,7 @@ func TestAvailableDevices(t *testing.T) { if command == "lsblk" { if strings.Index(name, "sdb") != -1 { // /dev/sdb has a partition - return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL="" + return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME="" NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil } return "", nil @@ -120,44 +120,50 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil &inventory.LocalDisk{Name: "sdb"}, &inventory.LocalDisk{Name: "sdc"}, &inventory.LocalDisk{Name: "sdd"}, + &inventory.LocalDisk{Name: "nvme01"}, &inventory.LocalDisk{Name: "rda"}, &inventory.LocalDisk{Name: "rdb"}, } - // select all devices - mapping, err := getAvailableDevices(context, devices, "all", true) + + // select all devices, including nvme01 for metadata + mapping, err := getAvailableDevices(context, devices, "all", "nvme01", true) assert.Nil(t, err) - assert.Equal(t, 4, len(mapping.Entries)) + assert.Equal(t, 5, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) assert.Equal(t, -1, mapping.Entries["rda"].Data) assert.Equal(t, -1, mapping.Entries["rdb"].Data) + assert.Equal(t, -1, mapping.Entries["nvme01"].Data) + assert.NotNil(t, mapping.Entries["nvme01"].Metadata) + assert.Equal(t, 0, len(mapping.Entries["nvme01"].Metadata)) // select no devices both using and not using a filter - mapping, err = getAvailableDevices(context, devices, "", false) + mapping, err = getAvailableDevices(context, devices, "", "", false) assert.Nil(t, err) assert.Equal(t, 0, len(mapping.Entries)) - mapping, err = getAvailableDevices(context, devices, "", true) + mapping, err = getAvailableDevices(context, devices, "", "", true) assert.Nil(t, err) assert.Equal(t, 0, len(mapping.Entries)) - // select the sda devices - mapping, err = getAvailableDevices(context, devices, "^sd.$", true) + // select the sd* devices + mapping, err = getAvailableDevices(context, devices, "^sd.$", "", true) assert.Nil(t, err) assert.Equal(t, 2, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) // select an exact device - mapping, err = getAvailableDevices(context, devices, "sdd", false) + mapping, err = getAvailableDevices(context, devices, "sdd", "", false) assert.Nil(t, err) assert.Equal(t, 1, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sdd"].Data) // select all devices except those that have a prefix of "s" - mapping, err = getAvailableDevices(context, devices, "^[^s]", true) + mapping, err = getAvailableDevices(context, devices, "^[^s]", "", true) assert.Nil(t, err) - assert.Equal(t, 2, len(mapping.Entries)) + assert.Equal(t, 3, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["rda"].Data) assert.Equal(t, -1, mapping.Entries["rdb"].Data) + assert.Equal(t, -1, mapping.Entries["nvme01"].Data) } diff --git a/pkg/ceph/osd/device.go b/pkg/ceph/osd/device.go index 010b80940..37cc245cb 100644 --- a/pkg/ceph/osd/device.go +++ b/pkg/ceph/osd/device.go @@ -181,6 +181,14 @@ func partitionMetadata(context *clusterd.Context, info *MetadataDeviceInfo, conf savedScheme.Metadata.Device, savedScheme.Metadata.DiskUUID, info.Device, info.DiskUUID) } + // check one last time to make sure it's OK for us to format this metadata device + ownPartitions, fs, err := checkIfDeviceAvailable(context.Executor, info.Device) + if err != nil { + return fmt.Errorf("failed to get metadata device %s info: %+v", info.Device, err) + } else if fs != "" || !ownPartitions { + return fmt.Errorf("metadata device %s is already in use (not by rook). fs: %s, ownPartitions: %t", info.Device, fs, ownPartitions) + } + // zap/clear all existing partitions err = sys.RemovePartitions(info.Device, context.Executor) if err != nil { diff --git a/pkg/ceph/osd/device_test.go b/pkg/ceph/osd/device_test.go index ba87a943b..d0f48411c 100644 --- a/pkg/ceph/osd/device_test.go +++ b/pkg/ceph/osd/device_test.go @@ -125,7 +125,7 @@ func TestOverwriteRookOwnedPartitions(t *testing.T) { switch outputExecCount { case 0, 4: // we'll call this twice, once explicitly below to verify rook owns the partitions and a 2nd time within formatDevice assert.Equal(t, command, "lsblk") - output = `NAME="sda" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL="" + output = `NAME="sda" SIZE="65" TYPE="disk" PKNAME="" NAME="sda1" SIZE="30" TYPE="part" PKNAME="sda" NAME="sda2" SIZE="10" TYPE="part" PKNAME="sda" NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"` @@ -236,6 +236,45 @@ func TestPartitionBluestoreMetadata(t *testing.T) { assert.True(t, util.CreateSet(desiredIds).Equals(util.CreateSet([]string{"1", "2"}))) } +func TestPartitionBluestoreMetadataSafe(t *testing.T) { + // set up a temporary config directory that will be cleaned up after test + configDir, err := ioutil.TempDir("", "TestPartitionBluestoreMetadataSafe") + if err != nil { + t.Fatalf("failed to create temp config dir: %+v", err) + } + defer os.RemoveAll(configDir) + + nodeID := "node123" + etcdClient := util.NewMockEtcdClient() + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(name string, command string, args ...string) (string, error) { + if command == "df" { + // mock that the metadata device already has a filesytem, this should abort the partition effort + if strings.Index(name, "nvme01") != -1 { + return "/dev/nvme01 ext4", nil + } + } + + return "", nil + } + + context := &clusterd.Context{DirectContext: clusterd.DirectContext{EtcdClient: etcdClient, NodeID: nodeID}, Executor: executor, ConfigDir: configDir} + + // create metadata partition information for 1 OSD (sda) storing its metadata on device nvme01 + storeConfig := StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2} + metadata := NewMetadataDeviceInfo("nvme01") + e1 := NewPerfSchemeEntry(Bluestore) + e1.ID = 1 + e1.OsdUUID = uuid.Must(uuid.NewRandom()) + PopulateDistributedPerfSchemeEntry(e1, "sda", metadata, storeConfig) + + // attempt to perform the metadata device partition. this should fail because we should detect + // that the metadata device has a filesystem already (not safe to format) + err = partitionMetadata(context, metadata, configDir) + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "already in use (not by rook)")) +} + func TestPartitionOSD(t *testing.T) { testPartitionOSDHelper(t, StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2}) testPartitionOSDHelper(t, StoreConfig{StoreType: Filestore})