osd: include metadata devices when launching OSD daemon in Kubernetes
This commit is contained in:
1 parent
2838e638fe
commit
2359ca5249
4 files changed
+100
-38
No files matched your search
+35
-26
@@ -53,7 +53,7 @@ func Run(context *clusterd.Context, agent *OsdAgent) error {
|
||||
logger.Infof("creating and starting the osds")
|
||||
|
||||
// initialize the desired osds
|
||||
devices, err := getAvailableDevices(context, hardware.Disks, agent.devices, agent.usingDeviceFilter)
|
||||
devices, err := getAvailableDevices(context, hardware.Disks, agent.devices, agent.metadataDevice, agent.usingDeviceFilter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get available devices. %+v", err)
|
||||
}
|
||||
@@ -87,7 +87,7 @@ func Run(context *clusterd.Context, agent *OsdAgent) error {
|
||||
}
|
||||
|
||||
func getAvailableDevices(context *clusterd.Context, devices []*inventory.LocalDisk, desiredDevices string,
|
||||
usingDeviceFilter bool) (*DeviceOsdMapping, error) {
|
||||
metadataDevice string, usingDeviceFilter bool) (*DeviceOsdMapping, error) {
|
||||
|
||||
var deviceList []string
|
||||
if !usingDeviceFilter {
|
||||
@@ -103,33 +103,42 @@ func getAvailableDevices(context *clusterd.Context, devices []*inventory.LocalDi
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err)
|
||||
}
|
||||
if fs == "" && ownPartitions {
|
||||
if desiredDevices == "all" {
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
|
||||
} else if desiredDevices != "" {
|
||||
var matched bool
|
||||
var err error
|
||||
if usingDeviceFilter {
|
||||
// the desired devices is a regular expression
|
||||
matched, err = regexp.Match(desiredDevices, []byte(device.Name))
|
||||
} else {
|
||||
for i := range deviceList {
|
||||
if device.Name == deviceList[i] {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
|
||||
if fs != "" || !ownPartitions {
|
||||
// not OK to use the device because it has a filesystem or rook doesn't own all its partitions
|
||||
logger.Infof("skipping device %s that is in use (not by rook). fs: %s, ownPartitions: %t", device.Name, fs, ownPartitions)
|
||||
continue
|
||||
}
|
||||
|
||||
if metadataDevice != "" && metadataDevice == device.Name {
|
||||
// current device is desired as the metadata device
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}}
|
||||
} else if desiredDevices == "all" {
|
||||
// user has specified all devices, use the current one for data
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
|
||||
} else if desiredDevices != "" {
|
||||
var matched bool
|
||||
var err error
|
||||
if usingDeviceFilter {
|
||||
// the desired devices is a regular expression
|
||||
matched, err = regexp.Match(desiredDevices, []byte(device.Name))
|
||||
} else {
|
||||
for i := range deviceList {
|
||||
if device.Name == deviceList[i] {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil && matched {
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
|
||||
} else {
|
||||
logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err)
|
||||
}
|
||||
|
||||
} else {
|
||||
logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name)
|
||||
}
|
||||
|
||||
if err == nil && matched {
|
||||
// the current device matches the user specifies filter/list, use it for data
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
|
||||
} else {
|
||||
logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err)
|
||||
}
|
||||
} else {
|
||||
logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+17
-11
@@ -94,7 +94,7 @@ func TestAvailableDevices(t *testing.T) {
|
||||
if command == "lsblk" {
|
||||
if strings.Index(name, "sdb") != -1 {
|
||||
// /dev/sdb has a partition
|
||||
return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL=""
|
||||
return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME=""
|
||||
NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
|
||||
}
|
||||
return "", nil
|
||||
@@ -120,44 +120,50 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
|
||||
&inventory.LocalDisk{Name: "sdb"},
|
||||
&inventory.LocalDisk{Name: "sdc"},
|
||||
&inventory.LocalDisk{Name: "sdd"},
|
||||
&inventory.LocalDisk{Name: "nvme01"},
|
||||
&inventory.LocalDisk{Name: "rda"},
|
||||
&inventory.LocalDisk{Name: "rdb"},
|
||||
}
|
||||
// select all devices
|
||||
mapping, err := getAvailableDevices(context, devices, "all", true)
|
||||
|
||||
// select all devices, including nvme01 for metadata
|
||||
mapping, err := getAvailableDevices(context, devices, "all", "nvme01", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 4, len(mapping.Entries))
|
||||
assert.Equal(t, 5, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["rda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["rdb"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["nvme01"].Data)
|
||||
assert.NotNil(t, mapping.Entries["nvme01"].Metadata)
|
||||
assert.Equal(t, 0, len(mapping.Entries["nvme01"].Metadata))
|
||||
|
||||
// select no devices both using and not using a filter
|
||||
mapping, err = getAvailableDevices(context, devices, "", false)
|
||||
mapping, err = getAvailableDevices(context, devices, "", "", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(mapping.Entries))
|
||||
|
||||
mapping, err = getAvailableDevices(context, devices, "", true)
|
||||
mapping, err = getAvailableDevices(context, devices, "", "", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(mapping.Entries))
|
||||
|
||||
// select the sda devices
|
||||
mapping, err = getAvailableDevices(context, devices, "^sd.$", true)
|
||||
// select the sd* devices
|
||||
mapping, err = getAvailableDevices(context, devices, "^sd.$", "", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
|
||||
// select an exact device
|
||||
mapping, err = getAvailableDevices(context, devices, "sdd", false)
|
||||
mapping, err = getAvailableDevices(context, devices, "sdd", "", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
|
||||
// select all devices except those that have a prefix of "s"
|
||||
mapping, err = getAvailableDevices(context, devices, "^[^s]", true)
|
||||
mapping, err = getAvailableDevices(context, devices, "^[^s]", "", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(mapping.Entries))
|
||||
assert.Equal(t, 3, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["rda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["rdb"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["nvme01"].Data)
|
||||
}
|
||||
@@ -181,6 +181,14 @@ func partitionMetadata(context *clusterd.Context, info *MetadataDeviceInfo, conf
|
||||
savedScheme.Metadata.Device, savedScheme.Metadata.DiskUUID, info.Device, info.DiskUUID)
|
||||
}
|
||||
|
||||
// check one last time to make sure it's OK for us to format this metadata device
|
||||
ownPartitions, fs, err := checkIfDeviceAvailable(context.Executor, info.Device)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get metadata device %s info: %+v", info.Device, err)
|
||||
} else if fs != "" || !ownPartitions {
|
||||
return fmt.Errorf("metadata device %s is already in use (not by rook). fs: %s, ownPartitions: %t", info.Device, fs, ownPartitions)
|
||||
}
|
||||
|
||||
// zap/clear all existing partitions
|
||||
err = sys.RemovePartitions(info.Device, context.Executor)
|
||||
if err != nil {
|
||||
|
||||
@@ -125,7 +125,7 @@ func TestOverwriteRookOwnedPartitions(t *testing.T) {
|
||||
switch outputExecCount {
|
||||
case 0, 4: // we'll call this twice, once explicitly below to verify rook owns the partitions and a 2nd time within formatDevice
|
||||
assert.Equal(t, command, "lsblk")
|
||||
output = `NAME="sda" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL=""
|
||||
output = `NAME="sda" SIZE="65" TYPE="disk" PKNAME=""
|
||||
NAME="sda1" SIZE="30" TYPE="part" PKNAME="sda"
|
||||
NAME="sda2" SIZE="10" TYPE="part" PKNAME="sda"
|
||||
NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"`
|
||||
@@ -236,6 +236,45 @@ func TestPartitionBluestoreMetadata(t *testing.T) {
|
||||
assert.True(t, util.CreateSet(desiredIds).Equals(util.CreateSet([]string{"1", "2"})))
|
||||
}
|
||||
|
||||
func TestPartitionBluestoreMetadataSafe(t *testing.T) {
|
||||
// set up a temporary config directory that will be cleaned up after test
|
||||
configDir, err := ioutil.TempDir("", "TestPartitionBluestoreMetadataSafe")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create temp config dir: %+v", err)
|
||||
}
|
||||
defer os.RemoveAll(configDir)
|
||||
|
||||
nodeID := "node123"
|
||||
etcdClient := util.NewMockEtcdClient()
|
||||
executor := &exectest.MockExecutor{}
|
||||
executor.MockExecuteCommandWithOutput = func(name string, command string, args ...string) (string, error) {
|
||||
if command == "df" {
|
||||
// mock that the metadata device already has a filesytem, this should abort the partition effort
|
||||
if strings.Index(name, "nvme01") != -1 {
|
||||
return "/dev/nvme01 ext4", nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
context := &clusterd.Context{DirectContext: clusterd.DirectContext{EtcdClient: etcdClient, NodeID: nodeID}, Executor: executor, ConfigDir: configDir}
|
||||
|
||||
// create metadata partition information for 1 OSD (sda) storing its metadata on device nvme01
|
||||
storeConfig := StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2}
|
||||
metadata := NewMetadataDeviceInfo("nvme01")
|
||||
e1 := NewPerfSchemeEntry(Bluestore)
|
||||
e1.ID = 1
|
||||
e1.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
PopulateDistributedPerfSchemeEntry(e1, "sda", metadata, storeConfig)
|
||||
|
||||
// attempt to perform the metadata device partition. this should fail because we should detect
|
||||
// that the metadata device has a filesystem already (not safe to format)
|
||||
err = partitionMetadata(context, metadata, configDir)
|
||||
assert.NotNil(t, err)
|
||||
assert.True(t, strings.Contains(err.Error(), "already in use (not by rook)"))
|
||||
}
|
||||
|
||||
func TestPartitionOSD(t *testing.T) {
|
||||
testPartitionOSDHelper(t, StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2})
|
||||
testPartitionOSDHelper(t, StoreConfig{StoreType: Filestore})
|
||||
|
||||
Reference in new issue
Block a user