detect rook partitions

(cherry picked from commit 0f5552389eba6dad8babab027a05a5466517177f)
This commit is contained in:
Travis Nielsen authored and Jared Watts committed 2017-01-20 16:48:17 -08:00
1 parent 35ab726046
commit 5d76276f29
9 files changed
+194 -54

No files matched your search

+3 -2
View File
@@ -33,6 +33,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/sys"
"github.com/stretchr/testify/assert"
)
@@ -63,8 +64,8 @@ func TestGetNodesHandler(t *testing.T) {
// add the disks to etcd
disks := []inventory.Disk{
inventory.Disk{Type: inventory.DiskType, Size: 50, Rotational: true, Available: true},
inventory.Disk{Type: inventory.DiskType, Size: 100, Rotational: false, Available: true},
inventory.Disk{Type: sys.DiskType, Size: 50, Rotational: true, Available: true},
inventory.Disk{Type: sys.DiskType, Size: 100, Rotational: false, Available: true},
}
output, _ := json.Marshal(disks)
etcdClient.SetValue(path.Join(inventory.NodesConfigKey, nodeID, "disks"), string(output))
+1 -1
View File
@@ -122,7 +122,7 @@ func TestOSDAgentWithDevices(t *testing.T) {
assert.Equal(t, 0, agent.configCounter)
assert.Equal(t, 8, execCount)
assert.Equal(t, 2, outputExecCount)
assert.Equal(t, 4, outputExecCount)
assert.Equal(t, 2, startCount)
assert.Equal(t, 2, len(agent.osdProc), fmt.Sprintf("procs=%+v", agent.osdProc))
+44 -30
View File
@@ -40,7 +40,6 @@ import (
const (
DevicesValue = "devices"
ForceFormatValue = "forceFormat"
sgdisk = "sgdisk"
cephOsdKey = mon.CephKey + "/osd"
desiredOsdRootKey = cephOsdKey + "/" + clusterd.DesiredKey + "/%s"
deviceDesiredKey = desiredOsdRootKey + "/device"
@@ -69,33 +68,62 @@ type Device struct {
// format the given device for usage by an OSD
func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool) error {
// format the current volume
dangerousToFormat := false
// check if partitions belong to rook
partitions, _, err := sys.GetDevicePartitions(config.deviceName, context.Executor)
if err != nil {
return fmt.Errorf("failed to get %s partitions. %+v", config.deviceName, err)
}
if !rookOwnsPartitions(partitions) {
dangerousToFormat = true
if forceFormat {
logger.Warningf("device %s is being formatted!! partitions: %+v", config.deviceName, partitions)
} else {
logger.Warningf("device %s has partitions that will not be formatted. Skipping device.", config.deviceName)
}
}
// check if there is a file system on the device
devFS, err := sys.GetDeviceFilesystems(config.deviceName, context.Executor)
if err != nil {
return fmt.Errorf("failed to get device %s filesystem: %+v", config.deviceName, err)
}
if devFS != "" && forceFormat {
// there's a filesystem on the device, but the user has specified to force a format. give a warning about that.
logger.Warningf("device %s already formatted with %s, but forcing a format!!!", config.deviceName, devFS)
if devFS != "" {
dangerousToFormat = true
if forceFormat {
// there's a filesystem on the device, but the user has specified to force a format. give a warning about that.
logger.Warningf("device %s already formatted with %s, but forcing a format!!!", config.deviceName, devFS)
} else {
// disk is already formatted and the user doesn't want to force it, but we require partitioning
return fmt.Errorf("device %s already formatted with %s", config.deviceName, devFS)
}
}
if devFS == "" || forceFormat {
logger.Infof("Partitioning device %s for bluestore", config.deviceName)
// format the device
if !dangerousToFormat || forceFormat {
err := partitionBluestoreDevice(context, config)
if err != nil {
return fmt.Errorf("failed to partion device %s. %v", config.deviceName, err)
}
} else {
// disk is already formatted and the user doesn't want to force it, but we require partitioning
return fmt.Errorf("device %s already formatted with %s", config.deviceName, devFS)
}
return nil
}
func rookOwnsPartitions(partitions []*sys.Partition) bool {
// if there are partitions, they must all have the rook osd label
for _, p := range partitions {
if !strings.HasPrefix(p.Name, "ROOK-OSD") {
return false
}
}
// if there are no partitions, or the partitions are all from OSDs, then rook owns the device
return true
}
// Partitions a device for use by a bluestore osd.
// If there are any partitions or formatting already on the device, it will be wiped.
func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) error {
@@ -104,14 +132,7 @@ func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) erro
return fmt.Errorf("failed to get device size for osd %d. %+v", config.id, err)
}
cmd := fmt.Sprintf("zap %s osd%d", config.deviceName, config.id)
err = context.Executor.ExecuteCommand(cmd, sgdisk, "--zap-all", "/dev/"+config.deviceName)
if err != nil {
return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", config.deviceName, err)
}
cmd = fmt.Sprintf("clear %s osd%d", config.deviceName, config.id)
err = context.Executor.ExecuteCommand(cmd, sgdisk, "--clear", "--mbrtogpt", "/dev/"+config.deviceName)
err = sys.RemovePartitions(config.deviceName, context.Executor)
if err != nil {
return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", config.deviceName, err)
}
@@ -122,15 +143,8 @@ func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) erro
}
config.diskUUID = scheme.DiskUUID
// get args for creating partitions
args := scheme.GetArgs(config.deviceName)
if err != nil {
return fmt.Errorf("failed to get partition args. %+v", err)
}
// execute the partition command
cmd = fmt.Sprintf("partition %s osd%d", config.deviceName, config.id)
err = context.Executor.ExecuteCommand(cmd, sgdisk, args...)
// create the partitions
err = sys.CreatePartitions(config.deviceName, scheme.GetArgs(config.deviceName), context.Executor)
if err != nil {
return fmt.Errorf("failed to partition /dev/%s. %+v", config.deviceName, err)
}
+4 -4
View File
@@ -72,7 +72,7 @@ func loadDisksConfig(nodeConfig *NodeConfig, rawDisks string) error {
func toClusterDisks(devices []*LocalDisk) []*Disk {
var disks []*Disk
for _, device := range devices {
if device.Type == DiskType || device.Type == SSDType {
if device.Type == sys.DiskType || device.Type == sys.SSDType {
disk := &Disk{
Type: device.Type,
Size: device.Size,
@@ -87,7 +87,7 @@ func toClusterDisks(devices []*LocalDisk) []*Disk {
// check whether a device is available for use by bluestore
func getDeviceAvailable(device *LocalDisk) bool {
return device.Parent == "" && device.Type == DiskType && device.FileSystem == ""
return device.Parent == "" && device.Type == sys.DiskType && device.FileSystem == ""
}
// Discover all the details of devices available on the local node
@@ -108,14 +108,14 @@ func discoverDevices(executor exec.Executor) ([]*LocalDisk, error) {
}
diskType, ok := diskProps["TYPE"]
if !ok || (diskType != SSDType && diskType != DiskType && diskType != PartType) {
if !ok || (diskType != sys.SSDType && diskType != sys.DiskType && diskType != sys.PartType) {
// unsupported disk type, just continue
continue
}
// get the UUID for disks
var diskUUID string
if diskType != "part" {
if diskType != sys.PartType {
diskUUID, err = sys.GetDiskUUID(d, executor)
if err != nil {
logger.Warningf("skipping device %s with an unknown uuid. %+v", d, err)
+8 -6
View File
@@ -22,6 +22,7 @@ import (
"github.com/rook/rook/pkg/util"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/rook/rook/pkg/util/sys"
"github.com/stretchr/testify/assert"
)
@@ -29,9 +30,9 @@ func TestSerializeClusterDisks(t *testing.T) {
nodeID := "df1c87e8266843f2ab822c0d72f584d3"
etcdClient := &util.MockEtcdClient{}
d1 := &LocalDisk{Name: "sda", UUID: "u1", Size: 23, Rotational: true, Readonly: false,
FileSystem: "btrfs", MountPoint: "/mnt/abc", Type: DiskType, HasChildren: true}
FileSystem: "btrfs", MountPoint: "/mnt/abc", Type: sys.DiskType, HasChildren: true}
d2 := &LocalDisk{Name: "sdb", UUID: "u2", Size: 24, Rotational: true, Readonly: false,
Type: DiskType, HasChildren: true}
Type: sys.DiskType, HasChildren: true}
err := storeDevices(etcdClient, nodeID, []*LocalDisk{d1, d2})
assert.Nil(t, err)
@@ -65,16 +66,17 @@ func TestAvailableDisks(t *testing.T) {
assert.Equal(t, 0, len(disks))
// multiple available disks
d2 := &LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true}
d3 := &LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true}
d2 := &LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true}
d3 := &LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true}
disks = GetAvailableDevices([]*LocalDisk{d1, d2, d3})
assert.Equal(t, 2, len(disks))
assert.Equal(t, "sdb", disks[0])
assert.Equal(t, "sdc", disks[1])
// partitions don't result in more available devices
d4 := &LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: PartType, HasChildren: true}
d5 := &LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: PartType, HasChildren: true}
d4 := &LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true}
d5 := &LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true}
disks = GetAvailableDevices([]*LocalDisk{d1, d2, d3, d4, d5})
assert.Equal(t, 2, len(disks))
assert.Equal(t, "sdb", disks[0])
-6
View File
@@ -17,12 +17,6 @@ package inventory
import "time"
const (
DiskType = "disk"
SSDType = "ssd"
PartType = "part"
)
type Config struct {
Nodes map[string]*NodeConfig `json:"nodes"`
Local *Hardware `json:"local"`
+3 -2
View File
@@ -21,6 +21,7 @@ import (
"time"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/sys"
"github.com/stretchr/testify/assert"
)
@@ -61,8 +62,8 @@ func TestLoadHardwareConfig(t *testing.T) {
etcdClient := util.NewMockEtcdClient()
// setup disk info in etcd
d1 := &LocalDisk{Name: "sda", UUID: "uuid1", Size: 10737418240, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true}
d2 := &LocalDisk{Name: "sda2", UUID: "uuid2", Size: 2097152, Rotational: false, Readonly: true, Type: PartType, HasChildren: false}
d1 := &LocalDisk{Name: "sda", UUID: "uuid1", Size: 10737418240, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true}
d2 := &LocalDisk{Name: "sda2", UUID: "uuid2", Size: 2097152, Rotational: false, Readonly: true, Type: sys.PartType, HasChildren: false}
err := storeDevices(etcdClient, nodeID, []*LocalDisk{d1, d2})
assert.Nil(t, err)
+78 -3
View File
@@ -22,16 +22,31 @@ import (
"path/filepath"
"strings"
"strconv"
"github.com/google/uuid"
"github.com/rook/rook/pkg/util/exec"
)
const (
DiskType = "disk"
SSDType = "ssd"
PartType = "part"
sgdisk = "sgdisk"
)
type Partition struct {
Name string
Size uint64
Label string
}
// request the current user once and stash it in this global variable
var currentUser *user.User
func ListDevices(executor exec.Executor) ([]string, error) {
cmd := "lsblk all"
devices, err := executor.ExecuteCommandWithOutput(cmd, "lsblk", "--all", "-n", "-l", "--output", "KNAME")
devices, err := executor.ExecuteCommandWithOutput(cmd, "lsblk", "--all", "--noheadings", "--list", "--output", "KNAME")
if err != nil {
return nil, fmt.Errorf("failed to list all devices: %+v", err)
}
@@ -39,10 +54,49 @@ func ListDevices(executor exec.Executor) ([]string, error) {
return strings.Split(devices, "\n"), nil
}
func GetDevicePartitions(device string, executor exec.Executor) (partitions []*Partition, unusedSpace uint64, err error) {
cmd := fmt.Sprintf("lsblk /dev/%s", device)
output, err := executor.ExecuteCommandWithOutput(cmd, "lsblk", fmt.Sprintf("/dev/%s", device),
"--bytes", "--pairs", "--output", "NAME,SIZE,TYPE,PKNAME,PARTLABEL")
if err != nil {
return nil, 0, fmt.Errorf("failed to get device %s partitions. %+v", device, err)
}
partInfo := strings.Split(output, "\n")
var deviceSize uint64
var totalPartitionSize uint64
for _, info := range partInfo {
props := parseKeyValuePairString(info)
name := props["NAME"]
if name == device {
// found the main device
deviceSize, err = strconv.ParseUint(props["SIZE"], 10, 64)
if err != nil {
return nil, 0, fmt.Errorf("failed to get device %s size. %+v", device, err)
}
} else if props["PKNAME"] == device && props["TYPE"] == PartType {
// found a partition
p := &Partition{Name: name}
p.Label = props["PARTLABEL"]
p.Size, err = strconv.ParseUint(props["SIZE"], 10, 64)
if err != nil {
return nil, 0, fmt.Errorf("failed to get partition %s size. %+v", name, err)
}
totalPartitionSize += p.Size
partitions = append(partitions, p)
}
}
if deviceSize > 0 {
unusedSpace = deviceSize - totalPartitionSize
}
return partitions, unusedSpace, nil
}
func GetDeviceProperties(device string, executor exec.Executor) (map[string]string, error) {
cmd := fmt.Sprintf("lsblk /dev/%s", device)
output, err := executor.ExecuteCommandWithOutput(cmd, "lsblk", fmt.Sprintf("/dev/%s", device),
"-b", "-d", "-P", "-o", "SIZE,ROTA,RO,TYPE,PKNAME")
"--bytes", "--nodeps", "--pairs", "--output", "SIZE,ROTA,RO,TYPE,PKNAME")
if err != nil {
// try to get more information about the command error
cmdErr, ok := err.(*exec.CommandError)
@@ -69,6 +123,27 @@ func GetDeviceFilesystems(device string, executor exec.Executor) (string, error)
return parseDFOutput(device, output), nil
}
func RemovePartitions(device string, executor exec.Executor) error {
cmd := fmt.Sprintf("zap %s", device)
err := executor.ExecuteCommand(cmd, sgdisk, "--zap-all", "/dev/"+device)
if err != nil {
return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", device, err)
}
cmd = fmt.Sprintf("clear %s", device)
err = executor.ExecuteCommand(cmd, sgdisk, "--clear", "--mbrtogpt", "/dev/"+device)
if err != nil {
return fmt.Errorf("failed to clear partitions on /dev/%s: %+v", device, err)
}
return nil
}
func CreatePartitions(device string, args []string, executor exec.Executor) error {
cmd := fmt.Sprintf("partition %s", device)
return executor.ExecuteCommand(cmd, sgdisk, args...)
}
func FormatDevice(devicePath string, executor exec.Executor) error {
cmd := fmt.Sprintf("mkfs.ext4 %s", devicePath)
if err := executor.ExecuteCommand(cmd, "sudo", "mkfs.ext4", devicePath); err != nil {
@@ -82,7 +157,7 @@ func FormatDevice(devicePath string, executor exec.Executor) error {
func GetDiskUUID(device string, executor exec.Executor) (string, error) {
cmd := fmt.Sprintf("get disk %s uuid", device)
output, err := executor.ExecuteCommandWithOutput(cmd,
"sgdisk", "-p", fmt.Sprintf("/dev/%s", device))
sgdisk, "--print", fmt.Sprintf("/dev/%s", device))
if err != nil {
return "", err
}
+53
View File
@@ -120,3 +120,56 @@ func TestMountDeviceWithOptions(t *testing.T) {
// both fstype and options specified
MountDeviceWithOptions("/dev/abc1", "/tmp/mount1", "myfstype", "foo=bar,baz=biz", e)
}
func TestGetPartitions(t *testing.T) {
run := 0
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(actionName string, command string, arg ...string) (string, error) {
run++
switch {
case run == 1:
return `NAME="sdc" SIZE="100000" TYPE="disk" PKNAME="" PARTLABEL=""`, nil
case run == 2:
return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL=""
NAME="sdb2" SIZE="10" TYPE="part" PKNAME="sdb" PARTLABEL="ROOK-OSD0-DB"
NAME="sdb3" SIZE="20" TYPE="part" PKNAME="sdb" PARTLABEL="ROOK-OSD0-BLOCK"
NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="ROOK-OSD0-WAL"`, nil
case run == 3:
return `NAME="sda" SIZE="19818086400" TYPE="disk" PKNAME="" PARTLABEL=""
NAME="sda4" SIZE="1073741824" TYPE="part" PKNAME="sda" PARTLABEL="USR-B"
NAME="sda2" SIZE="2097152" TYPE="part" PKNAME="sda" PARTLABEL="BIOS-BOOT"
NAME="sda9" SIZE="17328766976" TYPE="part" PKNAME="sda" PARTLABEL="ROOT"
NAME="sda7" SIZE="67108864" TYPE="part" PKNAME="sda" PARTLABEL="OEM-CONFIG"
NAME="sda3" SIZE="1073741824" TYPE="part" PKNAME="sda" PARTLABEL="USR-A"
NAME="usr" SIZE="1065345024" TYPE="crypt" PKNAME="sda3" PARTLABEL=""
NAME="sda1" SIZE="134217728" TYPE="part" PKNAME="sda" PARTLABEL="EFI-SYSTEM"
NAME="sda6" SIZE="134217728" TYPE="part" PKNAME="sda" PARTLABEL="OEM"`, nil
case run == 4:
return "", nil
}
return "", nil
},
}
partitions, unused, err := GetDevicePartitions("sdc", executor)
assert.Nil(t, err)
assert.Equal(t, uint64(100000), unused)
assert.Equal(t, 0, len(partitions))
partitions, unused, err = GetDevicePartitions("sdb", executor)
assert.Nil(t, err)
assert.Equal(t, uint64(5), unused)
assert.Equal(t, 3, len(partitions))
assert.Equal(t, uint64(10), partitions[0].Size)
assert.Equal(t, "ROOK-OSD0-DB", partitions[0].Label)
assert.Equal(t, "sdb2", partitions[0].Name)
partitions, unused, err = GetDevicePartitions("sda", executor)
assert.Nil(t, err)
assert.Equal(t, uint64(0x400000), unused)
assert.Equal(t, 7, len(partitions))
partitions, unused, err = GetDevicePartitions("sdx", executor)
assert.Nil(t, err)
assert.Equal(t, 0, len(partitions))
}