diff --git a/pkg/api/handlers_test.go b/pkg/api/handlers_test.go index 9da2b32df..d2a2392c7 100644 --- a/pkg/api/handlers_test.go +++ b/pkg/api/handlers_test.go @@ -33,6 +33,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/clusterd/inventory" "github.com/rook/rook/pkg/util" + "github.com/rook/rook/pkg/util/sys" "github.com/stretchr/testify/assert" ) @@ -63,8 +64,8 @@ func TestGetNodesHandler(t *testing.T) { // add the disks to etcd disks := []inventory.Disk{ - inventory.Disk{Type: inventory.DiskType, Size: 50, Rotational: true, Available: true}, - inventory.Disk{Type: inventory.DiskType, Size: 100, Rotational: false, Available: true}, + inventory.Disk{Type: sys.DiskType, Size: 50, Rotational: true, Available: true}, + inventory.Disk{Type: sys.DiskType, Size: 100, Rotational: false, Available: true}, } output, _ := json.Marshal(disks) etcdClient.SetValue(path.Join(inventory.NodesConfigKey, nodeID, "disks"), string(output)) diff --git a/pkg/cephmgr/osd/agent_test.go b/pkg/cephmgr/osd/agent_test.go index deb8dbfcc..2673f28e9 100644 --- a/pkg/cephmgr/osd/agent_test.go +++ b/pkg/cephmgr/osd/agent_test.go @@ -122,7 +122,7 @@ func TestOSDAgentWithDevices(t *testing.T) { assert.Equal(t, 0, agent.configCounter) assert.Equal(t, 8, execCount) - assert.Equal(t, 2, outputExecCount) + assert.Equal(t, 4, outputExecCount) assert.Equal(t, 2, startCount) assert.Equal(t, 2, len(agent.osdProc), fmt.Sprintf("procs=%+v", agent.osdProc)) diff --git a/pkg/cephmgr/osd/device.go b/pkg/cephmgr/osd/device.go index 53829f69a..7fbc6dcfd 100644 --- a/pkg/cephmgr/osd/device.go +++ b/pkg/cephmgr/osd/device.go @@ -40,7 +40,6 @@ import ( const ( DevicesValue = "devices" ForceFormatValue = "forceFormat" - sgdisk = "sgdisk" cephOsdKey = mon.CephKey + "/osd" desiredOsdRootKey = cephOsdKey + "/" + clusterd.DesiredKey + "/%s" deviceDesiredKey = desiredOsdRootKey + "/device" @@ -69,33 +68,62 @@ type Device struct { // format the given device for usage by an OSD func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool) error { - // format the current volume + dangerousToFormat := false + + // check if partitions belong to rook + partitions, _, err := sys.GetDevicePartitions(config.deviceName, context.Executor) + if err != nil { + return fmt.Errorf("failed to get %s partitions. %+v", config.deviceName, err) + } + if !rookOwnsPartitions(partitions) { + dangerousToFormat = true + if forceFormat { + logger.Warningf("device %s is being formatted!! partitions: %+v", config.deviceName, partitions) + } else { + logger.Warningf("device %s has partitions that will not be formatted. Skipping device.", config.deviceName) + } + } + + // check if there is a file system on the device devFS, err := sys.GetDeviceFilesystems(config.deviceName, context.Executor) if err != nil { return fmt.Errorf("failed to get device %s filesystem: %+v", config.deviceName, err) } - - if devFS != "" && forceFormat { - // there's a filesystem on the device, but the user has specified to force a format. give a warning about that. - logger.Warningf("device %s already formatted with %s, but forcing a format!!!", config.deviceName, devFS) + if devFS != "" { + dangerousToFormat = true + if forceFormat { + // there's a filesystem on the device, but the user has specified to force a format. give a warning about that. + logger.Warningf("device %s already formatted with %s, but forcing a format!!!", config.deviceName, devFS) + } else { + // disk is already formatted and the user doesn't want to force it, but we require partitioning + return fmt.Errorf("device %s already formatted with %s", config.deviceName, devFS) + } } - if devFS == "" || forceFormat { - logger.Infof("Partitioning device %s for bluestore", config.deviceName) - + // format the device + if !dangerousToFormat || forceFormat { err := partitionBluestoreDevice(context, config) if err != nil { return fmt.Errorf("failed to partion device %s. %v", config.deviceName, err) } - - } else { - // disk is already formatted and the user doesn't want to force it, but we require partitioning - return fmt.Errorf("device %s already formatted with %s", config.deviceName, devFS) } return nil } +func rookOwnsPartitions(partitions []*sys.Partition) bool { + + // if there are partitions, they must all have the rook osd label + for _, p := range partitions { + if !strings.HasPrefix(p.Name, "ROOK-OSD") { + return false + } + } + + // if there are no partitions, or the partitions are all from OSDs, then rook owns the device + return true +} + // Partitions a device for use by a bluestore osd. // If there are any partitions or formatting already on the device, it will be wiped. func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) error { @@ -104,14 +132,7 @@ func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) erro return fmt.Errorf("failed to get device size for osd %d. %+v", config.id, err) } - cmd := fmt.Sprintf("zap %s osd%d", config.deviceName, config.id) - err = context.Executor.ExecuteCommand(cmd, sgdisk, "--zap-all", "/dev/"+config.deviceName) - if err != nil { - return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", config.deviceName, err) - } - - cmd = fmt.Sprintf("clear %s osd%d", config.deviceName, config.id) - err = context.Executor.ExecuteCommand(cmd, sgdisk, "--clear", "--mbrtogpt", "/dev/"+config.deviceName) + err = sys.RemovePartitions(config.deviceName, context.Executor) if err != nil { return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", config.deviceName, err) } @@ -122,15 +143,8 @@ func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) erro } config.diskUUID = scheme.DiskUUID - // get args for creating partitions - args := scheme.GetArgs(config.deviceName) - if err != nil { - return fmt.Errorf("failed to get partition args. %+v", err) - } - - // execute the partition command - cmd = fmt.Sprintf("partition %s osd%d", config.deviceName, config.id) - err = context.Executor.ExecuteCommand(cmd, sgdisk, args...) + // create the partitions + err = sys.CreatePartitions(config.deviceName, scheme.GetArgs(config.deviceName), context.Executor) if err != nil { return fmt.Errorf("failed to partition /dev/%s. %+v", config.deviceName, err) } diff --git a/pkg/clusterd/inventory/disk.go b/pkg/clusterd/inventory/disk.go index 6fa7ef6ac..415952893 100644 --- a/pkg/clusterd/inventory/disk.go +++ b/pkg/clusterd/inventory/disk.go @@ -72,7 +72,7 @@ func loadDisksConfig(nodeConfig *NodeConfig, rawDisks string) error { func toClusterDisks(devices []*LocalDisk) []*Disk { var disks []*Disk for _, device := range devices { - if device.Type == DiskType || device.Type == SSDType { + if device.Type == sys.DiskType || device.Type == sys.SSDType { disk := &Disk{ Type: device.Type, Size: device.Size, @@ -87,7 +87,7 @@ func toClusterDisks(devices []*LocalDisk) []*Disk { // check whether a device is available for use by bluestore func getDeviceAvailable(device *LocalDisk) bool { - return device.Parent == "" && device.Type == DiskType && device.FileSystem == "" + return device.Parent == "" && device.Type == sys.DiskType && device.FileSystem == "" } // Discover all the details of devices available on the local node @@ -108,14 +108,14 @@ func discoverDevices(executor exec.Executor) ([]*LocalDisk, error) { } diskType, ok := diskProps["TYPE"] - if !ok || (diskType != SSDType && diskType != DiskType && diskType != PartType) { + if !ok || (diskType != sys.SSDType && diskType != sys.DiskType && diskType != sys.PartType) { // unsupported disk type, just continue continue } // get the UUID for disks var diskUUID string - if diskType != "part" { + if diskType != sys.PartType { diskUUID, err = sys.GetDiskUUID(d, executor) if err != nil { logger.Warningf("skipping device %s with an unknown uuid. %+v", d, err) diff --git a/pkg/clusterd/inventory/disk_test.go b/pkg/clusterd/inventory/disk_test.go index 732468106..7c2c6d60a 100644 --- a/pkg/clusterd/inventory/disk_test.go +++ b/pkg/clusterd/inventory/disk_test.go @@ -22,6 +22,7 @@ import ( "github.com/rook/rook/pkg/util" exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/rook/rook/pkg/util/sys" "github.com/stretchr/testify/assert" ) @@ -29,9 +30,9 @@ func TestSerializeClusterDisks(t *testing.T) { nodeID := "df1c87e8266843f2ab822c0d72f584d3" etcdClient := &util.MockEtcdClient{} d1 := &LocalDisk{Name: "sda", UUID: "u1", Size: 23, Rotational: true, Readonly: false, - FileSystem: "btrfs", MountPoint: "/mnt/abc", Type: DiskType, HasChildren: true} + FileSystem: "btrfs", MountPoint: "/mnt/abc", Type: sys.DiskType, HasChildren: true} d2 := &LocalDisk{Name: "sdb", UUID: "u2", Size: 24, Rotational: true, Readonly: false, - Type: DiskType, HasChildren: true} + Type: sys.DiskType, HasChildren: true} err := storeDevices(etcdClient, nodeID, []*LocalDisk{d1, d2}) assert.Nil(t, err) @@ -65,16 +66,17 @@ func TestAvailableDisks(t *testing.T) { assert.Equal(t, 0, len(disks)) // multiple available disks - d2 := &LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true} - d3 := &LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true} + d2 := &LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true} + d3 := &LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true} disks = GetAvailableDevices([]*LocalDisk{d1, d2, d3}) + assert.Equal(t, 2, len(disks)) assert.Equal(t, "sdb", disks[0]) assert.Equal(t, "sdc", disks[1]) // partitions don't result in more available devices - d4 := &LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: PartType, HasChildren: true} - d5 := &LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: PartType, HasChildren: true} + d4 := &LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true} + d5 := &LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true} disks = GetAvailableDevices([]*LocalDisk{d1, d2, d3, d4, d5}) assert.Equal(t, 2, len(disks)) assert.Equal(t, "sdb", disks[0]) diff --git a/pkg/clusterd/inventory/hardware.go b/pkg/clusterd/inventory/hardware.go index 9b6a9f329..e5e38c292 100644 --- a/pkg/clusterd/inventory/hardware.go +++ b/pkg/clusterd/inventory/hardware.go @@ -17,12 +17,6 @@ package inventory import "time" -const ( - DiskType = "disk" - SSDType = "ssd" - PartType = "part" -) - type Config struct { Nodes map[string]*NodeConfig `json:"nodes"` Local *Hardware `json:"local"` diff --git a/pkg/clusterd/inventory/node_test.go b/pkg/clusterd/inventory/node_test.go index 5152400fe..ac3f62c97 100644 --- a/pkg/clusterd/inventory/node_test.go +++ b/pkg/clusterd/inventory/node_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/rook/rook/pkg/util" + "github.com/rook/rook/pkg/util/sys" "github.com/stretchr/testify/assert" ) @@ -61,8 +62,8 @@ func TestLoadHardwareConfig(t *testing.T) { etcdClient := util.NewMockEtcdClient() // setup disk info in etcd - d1 := &LocalDisk{Name: "sda", UUID: "uuid1", Size: 10737418240, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true} - d2 := &LocalDisk{Name: "sda2", UUID: "uuid2", Size: 2097152, Rotational: false, Readonly: true, Type: PartType, HasChildren: false} + d1 := &LocalDisk{Name: "sda", UUID: "uuid1", Size: 10737418240, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true} + d2 := &LocalDisk{Name: "sda2", UUID: "uuid2", Size: 2097152, Rotational: false, Readonly: true, Type: sys.PartType, HasChildren: false} err := storeDevices(etcdClient, nodeID, []*LocalDisk{d1, d2}) assert.Nil(t, err) diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go index 76385cd7e..1c2df2a50 100644 --- a/pkg/util/sys/device.go +++ b/pkg/util/sys/device.go @@ -22,16 +22,31 @@ import ( "path/filepath" "strings" + "strconv" + "github.com/google/uuid" "github.com/rook/rook/pkg/util/exec" ) +const ( + DiskType = "disk" + SSDType = "ssd" + PartType = "part" + sgdisk = "sgdisk" +) + +type Partition struct { + Name string + Size uint64 + Label string +} + // request the current user once and stash it in this global variable var currentUser *user.User func ListDevices(executor exec.Executor) ([]string, error) { cmd := "lsblk all" - devices, err := executor.ExecuteCommandWithOutput(cmd, "lsblk", "--all", "-n", "-l", "--output", "KNAME") + devices, err := executor.ExecuteCommandWithOutput(cmd, "lsblk", "--all", "--noheadings", "--list", "--output", "KNAME") if err != nil { return nil, fmt.Errorf("failed to list all devices: %+v", err) } @@ -39,10 +54,49 @@ func ListDevices(executor exec.Executor) ([]string, error) { return strings.Split(devices, "\n"), nil } +func GetDevicePartitions(device string, executor exec.Executor) (partitions []*Partition, unusedSpace uint64, err error) { + cmd := fmt.Sprintf("lsblk /dev/%s", device) + output, err := executor.ExecuteCommandWithOutput(cmd, "lsblk", fmt.Sprintf("/dev/%s", device), + "--bytes", "--pairs", "--output", "NAME,SIZE,TYPE,PKNAME,PARTLABEL") + if err != nil { + return nil, 0, fmt.Errorf("failed to get device %s partitions. %+v", device, err) + } + + partInfo := strings.Split(output, "\n") + var deviceSize uint64 + var totalPartitionSize uint64 + for _, info := range partInfo { + props := parseKeyValuePairString(info) + name := props["NAME"] + if name == device { + // found the main device + deviceSize, err = strconv.ParseUint(props["SIZE"], 10, 64) + if err != nil { + return nil, 0, fmt.Errorf("failed to get device %s size. %+v", device, err) + } + } else if props["PKNAME"] == device && props["TYPE"] == PartType { + // found a partition + p := &Partition{Name: name} + p.Label = props["PARTLABEL"] + p.Size, err = strconv.ParseUint(props["SIZE"], 10, 64) + if err != nil { + return nil, 0, fmt.Errorf("failed to get partition %s size. %+v", name, err) + } + totalPartitionSize += p.Size + partitions = append(partitions, p) + } + } + + if deviceSize > 0 { + unusedSpace = deviceSize - totalPartitionSize + } + return partitions, unusedSpace, nil +} + func GetDeviceProperties(device string, executor exec.Executor) (map[string]string, error) { cmd := fmt.Sprintf("lsblk /dev/%s", device) output, err := executor.ExecuteCommandWithOutput(cmd, "lsblk", fmt.Sprintf("/dev/%s", device), - "-b", "-d", "-P", "-o", "SIZE,ROTA,RO,TYPE,PKNAME") + "--bytes", "--nodeps", "--pairs", "--output", "SIZE,ROTA,RO,TYPE,PKNAME") if err != nil { // try to get more information about the command error cmdErr, ok := err.(*exec.CommandError) @@ -69,6 +123,27 @@ func GetDeviceFilesystems(device string, executor exec.Executor) (string, error) return parseDFOutput(device, output), nil } +func RemovePartitions(device string, executor exec.Executor) error { + cmd := fmt.Sprintf("zap %s", device) + err := executor.ExecuteCommand(cmd, sgdisk, "--zap-all", "/dev/"+device) + if err != nil { + return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", device, err) + } + + cmd = fmt.Sprintf("clear %s", device) + err = executor.ExecuteCommand(cmd, sgdisk, "--clear", "--mbrtogpt", "/dev/"+device) + if err != nil { + return fmt.Errorf("failed to clear partitions on /dev/%s: %+v", device, err) + } + + return nil +} + +func CreatePartitions(device string, args []string, executor exec.Executor) error { + cmd := fmt.Sprintf("partition %s", device) + return executor.ExecuteCommand(cmd, sgdisk, args...) +} + func FormatDevice(devicePath string, executor exec.Executor) error { cmd := fmt.Sprintf("mkfs.ext4 %s", devicePath) if err := executor.ExecuteCommand(cmd, "sudo", "mkfs.ext4", devicePath); err != nil { @@ -82,7 +157,7 @@ func FormatDevice(devicePath string, executor exec.Executor) error { func GetDiskUUID(device string, executor exec.Executor) (string, error) { cmd := fmt.Sprintf("get disk %s uuid", device) output, err := executor.ExecuteCommandWithOutput(cmd, - "sgdisk", "-p", fmt.Sprintf("/dev/%s", device)) + sgdisk, "--print", fmt.Sprintf("/dev/%s", device)) if err != nil { return "", err } diff --git a/pkg/util/sys/device_test.go b/pkg/util/sys/device_test.go index 92d04c63b..f9a231057 100644 --- a/pkg/util/sys/device_test.go +++ b/pkg/util/sys/device_test.go @@ -120,3 +120,56 @@ func TestMountDeviceWithOptions(t *testing.T) { // both fstype and options specified MountDeviceWithOptions("/dev/abc1", "/tmp/mount1", "myfstype", "foo=bar,baz=biz", e) } + +func TestGetPartitions(t *testing.T) { + run := 0 + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(actionName string, command string, arg ...string) (string, error) { + run++ + switch { + case run == 1: + return `NAME="sdc" SIZE="100000" TYPE="disk" PKNAME="" PARTLABEL=""`, nil + case run == 2: + return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL="" +NAME="sdb2" SIZE="10" TYPE="part" PKNAME="sdb" PARTLABEL="ROOK-OSD0-DB" +NAME="sdb3" SIZE="20" TYPE="part" PKNAME="sdb" PARTLABEL="ROOK-OSD0-BLOCK" +NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="ROOK-OSD0-WAL"`, nil + case run == 3: + return `NAME="sda" SIZE="19818086400" TYPE="disk" PKNAME="" PARTLABEL="" +NAME="sda4" SIZE="1073741824" TYPE="part" PKNAME="sda" PARTLABEL="USR-B" +NAME="sda2" SIZE="2097152" TYPE="part" PKNAME="sda" PARTLABEL="BIOS-BOOT" +NAME="sda9" SIZE="17328766976" TYPE="part" PKNAME="sda" PARTLABEL="ROOT" +NAME="sda7" SIZE="67108864" TYPE="part" PKNAME="sda" PARTLABEL="OEM-CONFIG" +NAME="sda3" SIZE="1073741824" TYPE="part" PKNAME="sda" PARTLABEL="USR-A" +NAME="usr" SIZE="1065345024" TYPE="crypt" PKNAME="sda3" PARTLABEL="" +NAME="sda1" SIZE="134217728" TYPE="part" PKNAME="sda" PARTLABEL="EFI-SYSTEM" +NAME="sda6" SIZE="134217728" TYPE="part" PKNAME="sda" PARTLABEL="OEM"`, nil + case run == 4: + return "", nil + } + return "", nil + }, + } + + partitions, unused, err := GetDevicePartitions("sdc", executor) + assert.Nil(t, err) + assert.Equal(t, uint64(100000), unused) + assert.Equal(t, 0, len(partitions)) + + partitions, unused, err = GetDevicePartitions("sdb", executor) + assert.Nil(t, err) + assert.Equal(t, uint64(5), unused) + assert.Equal(t, 3, len(partitions)) + assert.Equal(t, uint64(10), partitions[0].Size) + assert.Equal(t, "ROOK-OSD0-DB", partitions[0].Label) + assert.Equal(t, "sdb2", partitions[0].Name) + + partitions, unused, err = GetDevicePartitions("sda", executor) + assert.Nil(t, err) + assert.Equal(t, uint64(0x400000), unused) + assert.Equal(t, 7, len(partitions)) + + partitions, unused, err = GetDevicePartitions("sdx", executor) + assert.Nil(t, err) + assert.Equal(t, 0, len(partitions)) +}