osd: include metadata devices when launching OSD daemon in Kubernetes

This commit is contained in:
Jared Watts committed 2017-06-26 17:54:22 -07:00
1 parent 2838e638fe
commit 2359ca5249
4 files changed
+100 -38

No files matched your search

+35 -26
View File
@@ -53,7 +53,7 @@ func Run(context *clusterd.Context, agent *OsdAgent) error {
logger.Infof("creating and starting the osds")
// initialize the desired osds
devices, err := getAvailableDevices(context, hardware.Disks, agent.devices, agent.usingDeviceFilter)
devices, err := getAvailableDevices(context, hardware.Disks, agent.devices, agent.metadataDevice, agent.usingDeviceFilter)
if err != nil {
return fmt.Errorf("failed to get available devices. %+v", err)
}
@@ -87,7 +87,7 @@ func Run(context *clusterd.Context, agent *OsdAgent) error {
}
func getAvailableDevices(context *clusterd.Context, devices []*inventory.LocalDisk, desiredDevices string,
usingDeviceFilter bool) (*DeviceOsdMapping, error) {
metadataDevice string, usingDeviceFilter bool) (*DeviceOsdMapping, error) {
var deviceList []string
if !usingDeviceFilter {
@@ -103,33 +103,42 @@ func getAvailableDevices(context *clusterd.Context, devices []*inventory.LocalDi
if err != nil {
return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err)
}
if fs == "" && ownPartitions {
if desiredDevices == "all" {
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
} else if desiredDevices != "" {
var matched bool
var err error
if usingDeviceFilter {
// the desired devices is a regular expression
matched, err = regexp.Match(desiredDevices, []byte(device.Name))
} else {
for i := range deviceList {
if device.Name == deviceList[i] {
matched = true
break
}
if fs != "" || !ownPartitions {
// not OK to use the device because it has a filesystem or rook doesn't own all its partitions
logger.Infof("skipping device %s that is in use (not by rook). fs: %s, ownPartitions: %t", device.Name, fs, ownPartitions)
continue
}
if metadataDevice != "" && metadataDevice == device.Name {
// current device is desired as the metadata device
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}}
} else if desiredDevices == "all" {
// user has specified all devices, use the current one for data
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
} else if desiredDevices != "" {
var matched bool
var err error
if usingDeviceFilter {
// the desired devices is a regular expression
matched, err = regexp.Match(desiredDevices, []byte(device.Name))
} else {
for i := range deviceList {
if device.Name == deviceList[i] {
matched = true
break
}
}
if err == nil && matched {
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
} else {
logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err)
}
} else {
logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name)
}
if err == nil && matched {
// the current device matches the user specifies filter/list, use it for data
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
} else {
logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err)
}
} else {
logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name)
}
}
+17 -11
View File
@@ -94,7 +94,7 @@ func TestAvailableDevices(t *testing.T) {
if command == "lsblk" {
if strings.Index(name, "sdb") != -1 {
// /dev/sdb has a partition
return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL=""
return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME=""
NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
}
return "", nil
@@ -120,44 +120,50 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
&inventory.LocalDisk{Name: "sdb"},
&inventory.LocalDisk{Name: "sdc"},
&inventory.LocalDisk{Name: "sdd"},
&inventory.LocalDisk{Name: "nvme01"},
&inventory.LocalDisk{Name: "rda"},
&inventory.LocalDisk{Name: "rdb"},
}
// select all devices
mapping, err := getAvailableDevices(context, devices, "all", true)
// select all devices, including nvme01 for metadata
mapping, err := getAvailableDevices(context, devices, "all", "nvme01", true)
assert.Nil(t, err)
assert.Equal(t, 4, len(mapping.Entries))
assert.Equal(t, 5, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sda"].Data)
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
assert.Equal(t, -1, mapping.Entries["rda"].Data)
assert.Equal(t, -1, mapping.Entries["rdb"].Data)
assert.Equal(t, -1, mapping.Entries["nvme01"].Data)
assert.NotNil(t, mapping.Entries["nvme01"].Metadata)
assert.Equal(t, 0, len(mapping.Entries["nvme01"].Metadata))
// select no devices both using and not using a filter
mapping, err = getAvailableDevices(context, devices, "", false)
mapping, err = getAvailableDevices(context, devices, "", "", false)
assert.Nil(t, err)
assert.Equal(t, 0, len(mapping.Entries))
mapping, err = getAvailableDevices(context, devices, "", true)
mapping, err = getAvailableDevices(context, devices, "", "", true)
assert.Nil(t, err)
assert.Equal(t, 0, len(mapping.Entries))
// select the sda devices
mapping, err = getAvailableDevices(context, devices, "^sd.$", true)
// select the sd* devices
mapping, err = getAvailableDevices(context, devices, "^sd.$", "", true)
assert.Nil(t, err)
assert.Equal(t, 2, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sda"].Data)
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
// select an exact device
mapping, err = getAvailableDevices(context, devices, "sdd", false)
mapping, err = getAvailableDevices(context, devices, "sdd", "", false)
assert.Nil(t, err)
assert.Equal(t, 1, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
// select all devices except those that have a prefix of "s"
mapping, err = getAvailableDevices(context, devices, "^[^s]", true)
mapping, err = getAvailableDevices(context, devices, "^[^s]", "", true)
assert.Nil(t, err)
assert.Equal(t, 2, len(mapping.Entries))
assert.Equal(t, 3, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["rda"].Data)
assert.Equal(t, -1, mapping.Entries["rdb"].Data)
assert.Equal(t, -1, mapping.Entries["nvme01"].Data)
}
+8
View File
@@ -181,6 +181,14 @@ func partitionMetadata(context *clusterd.Context, info *MetadataDeviceInfo, conf
savedScheme.Metadata.Device, savedScheme.Metadata.DiskUUID, info.Device, info.DiskUUID)
}
// check one last time to make sure it's OK for us to format this metadata device
ownPartitions, fs, err := checkIfDeviceAvailable(context.Executor, info.Device)
if err != nil {
return fmt.Errorf("failed to get metadata device %s info: %+v", info.Device, err)
} else if fs != "" || !ownPartitions {
return fmt.Errorf("metadata device %s is already in use (not by rook). fs: %s, ownPartitions: %t", info.Device, fs, ownPartitions)
}
// zap/clear all existing partitions
err = sys.RemovePartitions(info.Device, context.Executor)
if err != nil {
+40 -1
View File
@@ -125,7 +125,7 @@ func TestOverwriteRookOwnedPartitions(t *testing.T) {
switch outputExecCount {
case 0, 4: // we'll call this twice, once explicitly below to verify rook owns the partitions and a 2nd time within formatDevice
assert.Equal(t, command, "lsblk")
output = `NAME="sda" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL=""
output = `NAME="sda" SIZE="65" TYPE="disk" PKNAME=""
NAME="sda1" SIZE="30" TYPE="part" PKNAME="sda"
NAME="sda2" SIZE="10" TYPE="part" PKNAME="sda"
NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"`
@@ -236,6 +236,45 @@ func TestPartitionBluestoreMetadata(t *testing.T) {
assert.True(t, util.CreateSet(desiredIds).Equals(util.CreateSet([]string{"1", "2"})))
}
func TestPartitionBluestoreMetadataSafe(t *testing.T) {
// set up a temporary config directory that will be cleaned up after test
configDir, err := ioutil.TempDir("", "TestPartitionBluestoreMetadataSafe")
if err != nil {
t.Fatalf("failed to create temp config dir: %+v", err)
}
defer os.RemoveAll(configDir)
nodeID := "node123"
etcdClient := util.NewMockEtcdClient()
executor := &exectest.MockExecutor{}
executor.MockExecuteCommandWithOutput = func(name string, command string, args ...string) (string, error) {
if command == "df" {
// mock that the metadata device already has a filesytem, this should abort the partition effort
if strings.Index(name, "nvme01") != -1 {
return "/dev/nvme01 ext4", nil
}
}
return "", nil
}
context := &clusterd.Context{DirectContext: clusterd.DirectContext{EtcdClient: etcdClient, NodeID: nodeID}, Executor: executor, ConfigDir: configDir}
// create metadata partition information for 1 OSD (sda) storing its metadata on device nvme01
storeConfig := StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2}
metadata := NewMetadataDeviceInfo("nvme01")
e1 := NewPerfSchemeEntry(Bluestore)
e1.ID = 1
e1.OsdUUID = uuid.Must(uuid.NewRandom())
PopulateDistributedPerfSchemeEntry(e1, "sda", metadata, storeConfig)
// attempt to perform the metadata device partition. this should fail because we should detect
// that the metadata device has a filesystem already (not safe to format)
err = partitionMetadata(context, metadata, configDir)
assert.NotNil(t, err)
assert.True(t, strings.Contains(err.Error(), "already in use (not by rook)"))
}
func TestPartitionOSD(t *testing.T) {
testPartitionOSDHelper(t, StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2})
testPartitionOSDHelper(t, StoreConfig{StoreType: Filestore})