Merge pull request #1637 from rootfs/dedicated-osd-pod

add device discovery daemon

cmd/rook/discover.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import (
+	"fmt"
+	"github.com/rook/rook/pkg/daemon/discover"
+	"github.com/spf13/cobra"
+)
+
+var discoverCmd = &cobra.Command{
+	Use:    "discover",
+	Short:  "Discover devices",
+	Hidden: true,
+}
+
+func init() {
+	discoverCmd.RunE = startDiscover
+}
+
+func startDiscover(cmd *cobra.Command, args []string) error {
+	setLogLevel()
+
+	logStartupInfo(discoverCmd.Flags())
+
+	clientset, _, rookClientset, err := getClientset()
+	if err != nil {
+		terminateFatal(fmt.Errorf("failed to init k8s client. %+v\n", err))
+	}
+
+	context := createContext()
+	context.Clientset = clientset
+	context.RookClientset = rookClientset
+
+	err = discover.Run(context)
+	if err != nil {
+		terminateFatal(err)
+	}
+
+	return nil
+}

cmd/rook/main.go

@@ -100,6 +100,7 @@ func addCommands() {
 	rootCmd.AddCommand(mdsCmd)
 	rootCmd.AddCommand(agentCmd)
 	rootCmd.AddCommand(operatorCmd)
+	rootCmd.AddCommand(discoverCmd)
 }
 
 func setLogLevel() {

pkg/clusterd/context.go

@@ -19,6 +19,7 @@ import (
 	"github.com/coreos/pkg/capnslog"
 	rookclient "github.com/rook/rook/pkg/client/clientset/versioned"
 	"github.com/rook/rook/pkg/util/exec"
+	"github.com/rook/rook/pkg/util/sys"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/kubernetes"
 )
@@ -50,5 +51,5 @@ type Context struct {
 	NetworkInfo NetworkInfo
 
 	// The local devices detected on the node
-	Devices []*LocalDisk
+	Devices []*sys.LocalDisk
 }

pkg/clusterd/disk.go

@@ -30,22 +30,7 @@ var (
 	isRBD  = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$")
 )
 
-type LocalDisk struct {
-	Name        string `json:"name"`
-	ID          string `json:"id"`
-	UUID        string `json:"uuid"`
-	Size        uint64 `json:"size"`
-	Rotational  bool   `json:"rotational"`
-	Readonly    bool   `json:"readonly"`
-	FileSystem  string `json:"fileSystem"`
-	MountPoint  string `json:"mountPoint"`
-	Type        string `json:"type"`
-	Parent      string `json:"parent"`
-	HasChildren bool   `json:"hasChildren"`
-	Empty       bool   `json:"empty"`
-}
-
-func GetAvailableDevices(devices []*LocalDisk) []string {
+func GetAvailableDevices(devices []*sys.LocalDisk) []string {
 
 	var available []string
 	for _, device := range devices {
@@ -60,8 +45,8 @@ func GetAvailableDevices(devices []*LocalDisk) []string {
 }
 
 // check whether a device is completely empty
-func getDeviceEmpty(device *LocalDisk) bool {
-	return device.Parent == "" && (device.Type == sys.DiskType || device.Type == sys.SSDType || device.Type == sys.CryptType) && device.FileSystem == ""
+func getDeviceEmpty(device *sys.LocalDisk) bool {
+	return device.Parent == "" && (device.Type == sys.DiskType || device.Type == sys.SSDType || device.Type == sys.CryptType) && device.Filesystem == ""
 }
 
 func ignoreDevice(d string) bool {
@@ -69,9 +54,9 @@ func ignoreDevice(d string) bool {
 }
 
 // Discover all the details of devices available on the local node
-func DiscoverDevices(executor exec.Executor) ([]*LocalDisk, error) {
+func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) {
 
-	var disks []*LocalDisk
+	var disks []*sys.LocalDisk
 	devices, err := sys.ListDevices(executor)
 	if err != nil {
 		return nil, err
@@ -99,19 +84,20 @@ func DiscoverDevices(executor exec.Executor) ([]*LocalDisk, error) {
 		// get the UUID for disks
 		var diskUUID string
 		if diskType != sys.PartType {
-			diskUUID, err = sys.GetDiskUUID(d, executor)
+			diskUUID, err = sys.GetFSUUID(d, executor)
 			if err != nil {
 				logger.Warningf("skipping device %s with an unknown uuid. %+v", d, err)
 				continue
 			}
 		}
 
-		fs, err := sys.GetDeviceFilesystems(d, executor)
+		udevInfo, err := sys.GetUdevInfo(d, executor)
 		if err != nil {
-			return nil, err
+			logger.Warningf("failed to get udev info for device %s: %+v", d, err)
+			continue
 		}
 
-		disk := &LocalDisk{Name: d, UUID: diskUUID, FileSystem: fs}
+		disk := &sys.LocalDisk{Name: d, UUID: diskUUID}
 
 		if val, ok := diskProps["TYPE"]; ok {
 			disk.Type = val
@@ -137,6 +123,36 @@ func DiscoverDevices(executor exec.Executor) ([]*LocalDisk, error) {
 
 		disk.Empty = getDeviceEmpty(disk)
 
+		// parse udev info output
+		if val, ok := udevInfo["DEVLINKS"]; ok {
+			disk.DevLinks = val
+		}
+		if val, ok := udevInfo["ID_FS_UUID"]; ok {
+			disk.UUID = val
+		}
+		if val, ok := udevInfo["ID_FS_TYPE"]; ok {
+			disk.Filesystem = val
+		}
+		if val, ok := udevInfo["ID_SERIAL"]; ok {
+			disk.Serial = val
+		}
+
+		if val, ok := udevInfo["ID_VENDOR"]; ok {
+			disk.Vendor = val
+		}
+
+		if val, ok := udevInfo["ID_MODEL"]; ok {
+			disk.Model = val
+		}
+
+		if val, ok := udevInfo["ID_WWN_WITH_EXTENSION"]; ok {
+			disk.WWNVendorExtension = val
+		}
+
+		if val, ok := udevInfo["ID_WWN"]; ok {
+			disk.WWN = val
+		}
+
 		disks = append(disks, disk)
 	}
 

pkg/clusterd/disk_test.go

@@ -26,34 +26,34 @@ import (
 func TestAvailableDisks(t *testing.T) {
 
 	// no disks discovered for a node is an error
-	disks := GetAvailableDevices([]*LocalDisk{})
+	disks := GetAvailableDevices([]*sys.LocalDisk{})
 	assert.Equal(t, 0, len(disks))
 
 	// no available disks because of the formatting
-	d1 := &LocalDisk{Name: "sda", UUID: "myuuid1", Size: 123, Rotational: true, Readonly: false, FileSystem: "btrfs", MountPoint: "/mnt/abc", Type: sys.DiskType, HasChildren: true}
-	disks = GetAvailableDevices([]*LocalDisk{d1})
+	d1 := &sys.LocalDisk{Name: "sda", UUID: "myuuid1", Size: 123, Rotational: true, Readonly: false, Filesystem: "btrfs", Type: sys.DiskType, HasChildren: true}
+	disks = GetAvailableDevices([]*sys.LocalDisk{d1})
 	assert.Equal(t, 0, len(disks))
 
 	// multiple available disks
-	d2 := &LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true}
-	d3 := &LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true}
-	disks = GetAvailableDevices([]*LocalDisk{d1, d2, d3})
+	d2 := &sys.LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true}
+	d3 := &sys.LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true}
+	disks = GetAvailableDevices([]*sys.LocalDisk{d1, d2, d3})
 
 	assert.Equal(t, 2, len(disks))
 	assert.Equal(t, "sdb", disks[0])
 	assert.Equal(t, "sdc", disks[1])
 
 	// partitions don't result in more available devices
-	d4 := &LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true}
-	d5 := &LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true}
-	disks = GetAvailableDevices([]*LocalDisk{d1, d2, d3, d4, d5})
+	d4 := &sys.LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true}
+	d5 := &sys.LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true}
+	disks = GetAvailableDevices([]*sys.LocalDisk{d1, d2, d3, d4, d5})
 	assert.Equal(t, 2, len(disks))
 	assert.Equal(t, "sdb", disks[0])
 	assert.Equal(t, "sdc", disks[1])
 
 	// Crypt disk type results in available disk
-	d6 := &LocalDisk{Name: "sdd", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.CryptType, HasChildren: true}
-	disks = GetAvailableDevices([]*LocalDisk{d6})
+	d6 := &sys.LocalDisk{Name: "sdd", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.CryptType, HasChildren: true}
+	disks = GetAvailableDevices([]*sys.LocalDisk{d6})
 	assert.Equal(t, 1, len(disks))
 
 }

pkg/daemon/ceph/osd/agent_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	testop "github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
+	"github.com/rook/rook/pkg/util/sys"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -218,7 +219,7 @@ func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig rookalpha.StoreConf
 	_, _, sdxUUID := mockPartitionSchemeEntry(t, 23, "sdx", &storeConfig, agent.kv, agent.nodeName)
 
 	// note only sdx already has a UUID (it's been through partitioning)
-	context.Devices = []*clusterd.LocalDisk{
+	context.Devices = []*sys.LocalDisk{
 		{Name: "sdx", Size: 1234567890, UUID: sdxUUID},
 		{Name: "sdy", Size: 1234567890},
 	}
@@ -285,7 +286,7 @@ func TestOSDAgentNoDevices(t *testing.T) {
 
 	// set up expected ProcManager commands
 	context := &clusterd.Context{
-		Devices:   []*clusterd.LocalDisk{},
+		Devices:   []*sys.LocalDisk{},
 		Executor:  executor,
 		ConfigDir: configDir,
 	}
@@ -372,12 +373,12 @@ func createTestKeyring(t *testing.T, configRoot string, args []string) {
 func TestGetPartitionPerfScheme(t *testing.T) {
 	configDir, _ := ioutil.TempDir("", "")
 	defer os.RemoveAll(configDir)
-	context := &clusterd.Context{Devices: []*clusterd.LocalDisk{}, ConfigDir: configDir}
+	context := &clusterd.Context{Devices: []*sys.LocalDisk{}, ConfigDir: configDir}
 	test.CreateConfigDir(configDir)
 
 	// 3 disks: 2 for data and 1 for the metadata of both disks (2 WALs and 2 DBs)
 	a := &OsdAgent{devices: "sda,sdb", metadataDevice: "sdc", kv: mockKVStore(), nodeName: "a"}
-	context.Devices = []*clusterd.LocalDisk{
+	context.Devices = []*sys.LocalDisk{
 		{Name: "sda", Size: 107374182400}, // 100 GB
 		{Name: "sdb", Size: 107374182400}, // 100 GB
 		{Name: "sdc", Size: 44158681088},  // 1 MB (starting offset) + 2 * (576 MB + 20 GB) = 41.125 GB
@@ -412,7 +413,7 @@ func TestGetPartitionPerfScheme(t *testing.T) {
 			if command == "blkid" {
 				return "", nil
 			}
-			if command == "df" {
+			if command == "udevadm" {
 				return "", nil
 			}
 			return "", fmt.Errorf("unexpected command %s %v", command, args)
@@ -476,14 +477,14 @@ func TestGetPartitionSchemeDiskInUse(t *testing.T) {
 			if command == "blkid" {
 				return "", nil
 			}
-			if command == "df" {
+			if command == "udevadm" {
 				return "", nil
 			}
 			return "", fmt.Errorf("unexpected command %s %v", command, args)
 		},
 	}
 	context := &clusterd.Context{
-		Devices:   []*clusterd.LocalDisk{},
+		Devices:   []*sys.LocalDisk{},
 		ConfigDir: configDir,
 		Executor:  executor,
 	}
@@ -491,7 +492,7 @@ func TestGetPartitionSchemeDiskInUse(t *testing.T) {
 	a := &OsdAgent{devices: "sda", kv: mockKVStore()}
 	_, _, sdaUUID := mockPartitionSchemeEntry(t, 1, "sda", nil, a.kv, a.nodeName)
 
-	context.Devices = []*clusterd.LocalDisk{
+	context.Devices = []*sys.LocalDisk{
 		{Name: "sda", Size: 107374182400, UUID: sdaUUID}, // 100 GB
 	}
 
@@ -540,14 +541,14 @@ func TestGetPartitionSchemeDiskNameChanged(t *testing.T) {
 			if command == "blkid" {
 				return "", nil
 			}
-			if command == "df" {
+			if command == "udevadm" {
 				return "", nil
 			}
 			return "", fmt.Errorf("unexpected command %s %v", command, args)
 		},
 	}
 	context := &clusterd.Context{
-		Devices:   []*clusterd.LocalDisk{},
+		Devices:   []*sys.LocalDisk{},
 		ConfigDir: configDir,
 		Executor:  executor,
 	}
@@ -558,7 +559,7 @@ func TestGetPartitionSchemeDiskNameChanged(t *testing.T) {
 	// setup an existing partition schme with metadata on nvme01 and data on sda
 	_, metadataUUID, sdaUUID := mockDistributedPartitionScheme(t, 1, "nvme01", "sda", a.kv, a.nodeName)
 
-	context.Devices = []*clusterd.LocalDisk{
+	context.Devices = []*sys.LocalDisk{
 		{Name: "nvme01-changed", Size: 107374182400, UUID: metadataUUID},
 		{Name: "sda-changed", Size: 107374182400, UUID: sdaUUID},
 	}

pkg/daemon/ceph/osd/daemon.go

@@ -184,7 +184,7 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices string, metad
 		if device.Type == sys.PartType {
 			continue
 		}
-		ownPartitions, fs, err := checkIfDeviceAvailable(context.Executor, device.Name)
+		ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err)
 		}

pkg/daemon/ceph/osd/daemon_test.go

@@ -26,9 +26,43 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/operator/cluster/ceph/osd/config"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
+	"github.com/rook/rook/pkg/util/sys"
 	"github.com/stretchr/testify/assert"
 )
 
+const udevFSOutput = `
+DEVNAME=/dev/sdk
+DEVPATH=/devices/platform/host6/session2/target6:0:0/6:0:0:0/block/sdk
+DEVTYPE=disk
+ID_BUS=scsi
+ID_FS_TYPE=ext2
+ID_FS_USAGE=filesystem
+ID_FS_UUID=f2d38cba-37da-411d-b7ba-9a6696c58174
+ID_FS_UUID_ENC=f2d38cba-37da-411d-b7ba-9a6696c58174
+ID_FS_VERSION=1.0
+ID_MODEL=disk01
+ID_MODEL_ENC=disk01\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20
+ID_PATH=ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0
+ID_PATH_TAG=ip-127_0_0_1_3260-iscsi-iqn_2016-06_world_srv_storage_target01-lun-0
+ID_REVISION=4.0
+ID_SCSI=1
+ID_SCSI_SERIAL=d27e5d89-8829-468b-90ce-4ef8c02f07fe
+ID_SERIAL=36001405d27e5d898829468b90ce4ef8c
+ID_SERIAL_SHORT=6001405d27e5d898829468b90ce4ef8c
+ID_TARGET_PORT=0
+ID_TYPE=disk
+ID_VENDOR=LIO-ORG
+ID_VENDOR_ENC=LIO-ORG\x20
+ID_WWN=0x6001405d27e5d898
+ID_WWN_VENDOR_EXTENSION=0x829468b90ce4ef8c
+ID_WWN_WITH_EXTENSION=0x6001405d27e5d898829468b90ce4ef8c
+MAJOR=8
+MINOR=160
+SUBSYSTEM=block
+TAGS=:systemd:
+USEC_INITIALIZED=15981915740802
+`
+
 func TestRunDaemon(t *testing.T) {
 	configDir, _ := ioutil.TempDir("", "")
 	defer os.RemoveAll(configDir)
@@ -159,10 +193,10 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
 				// partition sdb1 has a label MY-PART
 				return "MY-PART", nil
 			}
-		} else if command == "df" {
+		} else if command == "udevadm" {
 			if strings.Index(name, "sdc") != -1 {
 				// /dev/sdc has a file system
-				return "/dev/sdc ext4", nil
+				return udevFSOutput, nil
 			}
 			return "", nil
 		}
@@ -171,7 +205,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
 	}
 
 	context := &clusterd.Context{Executor: executor}
-	context.Devices = []*clusterd.LocalDisk{
+	context.Devices = []*sys.LocalDisk{
 		{Name: "sda"},
 		{Name: "sdb"},
 		{Name: "sdc"},

pkg/daemon/ceph/osd/device.go

@@ -90,7 +90,7 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool
 	}
 
 	// check if partitions belong to rook
-	ownPartitions, devFS, err := checkIfDeviceAvailable(context.Executor, dataDetails.Device)
+	ownPartitions, devFS, err := sys.CheckIfDeviceAvailable(context.Executor, dataDetails.Device)
 	if err != nil {
 		return fmt.Errorf("failed to format device. %+v", err)
 	}
@@ -125,38 +125,6 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool
 	return nil
 }
 
-func checkIfDeviceAvailable(executor exec.Executor, name string) (bool, string, error) {
-	ownPartitions := true
-	partitions, _, err := sys.GetDevicePartitions(name, executor)
-	if err != nil {
-		return false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err)
-	}
-	if !rookOwnsPartitions(partitions) {
-		ownPartitions = false
-	}
-
-	// check if there is a file system on the device
-	devFS, err := sys.GetDeviceFilesystems(name, executor)
-	if err != nil {
-		return false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err)
-	}
-
-	return ownPartitions, devFS, nil
-}
-
-func rookOwnsPartitions(partitions []*sys.Partition) bool {
-
-	// if there are partitions, they must all have the rook osd label
-	for _, p := range partitions {
-		if !strings.HasPrefix(p.Label, "ROOK-OSD") {
-			return false
-		}
-	}
-
-	// if there are no partitions, or the partitions are all from rook OSDs, then rook owns the device
-	return true
-}
-
 // partitions a given device exclusively for metadata usage
 func partitionMetadata(context *clusterd.Context, info *config.MetadataDeviceInfo, kv *k8sutil.ConfigMapKVStore, storeName string) error {
 	if len(info.Partitions) == 0 {
@@ -181,7 +149,7 @@ func partitionMetadata(context *clusterd.Context, info *config.MetadataDeviceInf
 	}
 
 	// check one last time to make sure it's OK for us to format this metadata device
-	ownPartitions, fs, err := checkIfDeviceAvailable(context.Executor, info.Device)
+	ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, info.Device)
 	if err != nil {
 		return fmt.Errorf("failed to get metadata device %s info: %+v", info.Device, err)
 	} else if fs != "" || !ownPartitions {

pkg/daemon/ceph/osd/device_test.go

@@ -93,26 +93,7 @@ func TestOverwriteRookOwnedPartitions(t *testing.T) {
 	defer os.RemoveAll(configDir)
 
 	// set up mock execute so we can verify the partitioning happens on sda
-	execCount := 0
 	executor := &exectest.MockExecutor{}
-	executor.MockExecuteCommand = func(debug bool, name string, command string, args ...string) error {
-		logger.Infof("RUN %d for '%s'. %s %+v", execCount, name, command, args)
-		assert.Equal(t, "sgdisk", command)
-		switch execCount {
-		case 0:
-			assert.Equal(t, []string{"--zap-all", "/dev/sda"}, args)
-		case 1:
-			assert.Equal(t, []string{"--clear", "--mbrtogpt", "/dev/sda"}, args)
-		case 2:
-			assert.Equal(t, 11, len(args))
-			assert.Equal(t, "--change-name=1:ROOK-OSD1-WAL", args[1])
-			assert.Equal(t, "--change-name=2:ROOK-OSD1-DB", args[4])
-			assert.Equal(t, "--change-name=3:ROOK-OSD1-BLOCK", args[7])
-			assert.Equal(t, "/dev/sda", args[10])
-		}
-		execCount++
-		return nil
-	}
 
 	// set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem
 	outputExecCount := 0
@@ -120,23 +101,29 @@ func TestOverwriteRookOwnedPartitions(t *testing.T) {
 		logger.Infof("OUTPUT %d for %s. %s %+v", outputExecCount, name, command, args)
 		var output string
 		switch outputExecCount {
-		case 0, 4: // we'll call this twice, once explicitly below to verify rook owns the partitions and a 2nd time within formatDevice
+		case 0, 7: // we'll call this twice, once explicitly below to verify rook owns the partitions and a 2nd time within formatDevice
 			assert.Equal(t, command, "lsblk")
 			output = `NAME="sda" SIZE="65" TYPE="disk" PKNAME=""
 NAME="sda1" SIZE="30" TYPE="part" PKNAME="sda"
 NAME="sda2" SIZE="10" TYPE="part" PKNAME="sda"
 NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"`
-		case 1, 5:
-			assert.Equal(t, "blkid /dev/sda1", name)
-			output = "ROOK-OSD0-WAL"
-		case 2, 6:
-			assert.Equal(t, "blkid /dev/sda2", name)
-			output = "ROOK-OSD0-DB"
-		case 3, 7:
-			assert.Equal(t, "blkid /dev/sda3", name)
-			output = "ROOK-OSD0-BLOCK"
-		case 8:
-			assert.Equal(t, command, "df")
+		case 1, 8:
+			assert.Equal(t, "udevadm /dev/sda1", name)
+			output = "ID_PART_ENTRY_NAME=ROOK-OSD0-WAL"
+		case 2, 9:
+			assert.Equal(t, "get filesystem type for sda1", name)
+			output = ""
+		case 3, 10:
+			assert.Equal(t, "udevadm /dev/sda2", name)
+			output = "ID_PART_ENTRY_NAME=ROOK-OSD0-DB"
+		case 4, 11:
+			assert.Equal(t, "get filesystem type for sda2", name)
+			output = ""
+		case 5, 12:
+			assert.Equal(t, "udevadm /dev/sda3", name)
+			output = "ID_PART_ENTRY_NAME=ROOK-OSD0-BLOCK"
+		case 6, 13:
+			assert.Equal(t, "get filesystem type for sda3", name)
 			output = ""
 		}
 		outputExecCount++
@@ -152,7 +139,7 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"`
 	config.PopulateCollocatedPerfSchemeEntry(entry, "sda", storeConfig)
 
 	context := &clusterd.Context{Executor: executor, ConfigDir: configDir}
-	context.Devices = []*clusterd.LocalDisk{
+	context.Devices = []*sys.LocalDisk{
 		{Name: "sda", Size: 65},
 	}
 	config := &osdConfig{configRoot: configDir, rootPath: filepath.Join(configDir, "osd1"), id: entry.ID,
@@ -161,14 +148,13 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"`
 	// ensure that our mocking makes it look like rook owns the partitions on sda
 	partitions, _, err := sys.GetDevicePartitions("sda", context.Executor)
 	assert.Nil(t, err)
-	assert.True(t, rookOwnsPartitions(partitions))
+	assert.True(t, sys.RookOwnsPartitions(partitions))
 
 	// try to format the device.  even though the device has existing partitions, they are owned by rook, so it is safe
 	// to format and the format/partitioning will happen.
 	err = formatDevice(context, config, false, storeConfig)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, execCount)
-	assert.Equal(t, 9, outputExecCount)
+	assert.Equal(t, 15, outputExecCount)
 }
 
 func TestPartitionBluestoreMetadata(t *testing.T) {
@@ -238,10 +224,10 @@ func TestPartitionBluestoreMetadataSafe(t *testing.T) {
 	nodeID := "node123"
 	executor := &exectest.MockExecutor{}
 	executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) {
-		if command == "df" {
+		if command == "udevadm" {
 			// mock that the metadata device already has a filesytem, this should abort the partition effort
 			if strings.Index(name, "nvme01") != -1 {
-				return "/dev/nvme01 ext4", nil
+				return udevFSOutput, nil
 			}
 		}
 
@@ -320,7 +306,7 @@ func testPartitionOSDHelper(t *testing.T, storeConfig rookalpha.StoreConfig) {
 
 	// setup a context with 1 disk: sda
 	context := &clusterd.Context{Executor: executor, ConfigDir: configDir}
-	context.Devices = []*clusterd.LocalDisk{
+	context.Devices = []*sys.LocalDisk{
 		{Name: "sda", Size: 100},
 	}
 

pkg/daemon/discover/discover.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package discover to discover unused devices.
+package discover
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/coreos/pkg/capnslog"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	"github.com/rook/rook/pkg/util/sys"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	logger                                  = capnslog.NewPackageLogger("github.com/rook/rook", "rook-discover")
+	AppName                                 = "rook-discover"
+	NodeAttr                                = "rook.io/node"
+	LocalDiskCMData                         = "devices"
+	LocalDiskCMName                         = "local-device-"
+	probeInterval                           = 30 * time.Second
+	nodeName, namespace, lastDevice, cmName string
+	cm                                      *v1.ConfigMap
+)
+
+func Run(context *clusterd.Context) error {
+	if context == nil {
+		return fmt.Errorf("nil context")
+	}
+	nodeName = os.Getenv(k8sutil.NodeNameEnvVar)
+	namespace = os.Getenv(k8sutil.PodNamespaceEnvVar)
+	cmName = LocalDiskCMName + nodeName
+	tick := time.NewTicker(probeInterval).C
+	sigc := make(chan os.Signal, 1)
+	signal.Notify(sigc, syscall.SIGTERM)
+	for {
+		select {
+		case <-sigc:
+			logger.Infof("shutdown signal received, exiting...")
+			return nil
+		case <-tick:
+			err := updateDeviceCM(context)
+			if err != nil {
+				logger.Infof("failed to update device configmap: %v", err)
+			}
+		}
+	}
+}
+func updateDeviceCM(context *clusterd.Context) error {
+	logger.Infof("updating device configmap")
+	devices, err := probeDevices(context)
+	if err != nil {
+		logger.Infof("failed to probe devices: %v", err)
+		return err
+	}
+	deviceJson, err := json.Marshal(devices)
+	if err != nil {
+		logger.Infof("failed to marshal: %v", err)
+		return err
+	}
+	deviceStr := string(deviceJson)
+	if cm == nil {
+		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Get(cmName, metav1.GetOptions{})
+	}
+	if err == nil {
+		lastDevice = cm.Data[LocalDiskCMData]
+		logger.Debugf("last devices %s", lastDevice)
+	} else {
+		if !errors.IsNotFound(err) {
+			logger.Infof("failed to get configmap: %v", err)
+			return err
+		}
+
+		data := make(map[string]string, 1)
+		data[LocalDiskCMData] = deviceStr
+		// the map doesn't exist yet, create it now
+		cm = &v1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      cmName,
+				Namespace: namespace,
+				Labels: map[string]string{
+					k8sutil.AppAttr: AppName,
+					NodeAttr:        nodeName,
+				},
+			},
+			Data: data,
+		}
+		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Create(cm)
+		if err != nil {
+			logger.Infof("failed to create configmap: %v", err)
+			return fmt.Errorf("failed to create local device map %s: %+v", cmName, err)
+		}
+		lastDevice = deviceStr
+	}
+	if deviceStr != lastDevice {
+		data := make(map[string]string, 1)
+		data[LocalDiskCMData] = deviceStr
+		cm.Data = data
+		cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Update(cm)
+		if err != nil {
+			logger.Infof("failed to update configmap %s: %v", cmName, err)
+			return err
+		}
+	}
+	return nil
+}
+
+func probeDevices(context *clusterd.Context) ([]sys.LocalDisk, error) {
+	devices := make([]sys.LocalDisk, 0)
+	localDevices, err := clusterd.DiscoverDevices(context.Executor)
+	if err != nil {
+		return devices, fmt.Errorf("failed initial hardware discovery. %+v", err)
+	}
+	for _, device := range localDevices {
+		if device == nil {
+			continue
+		}
+		if device.Type == sys.PartType {
+			continue
+		}
+
+		partitions, _, err := sys.GetDevicePartitions(device.Name, context.Executor)
+		if err != nil {
+			logger.Infof("failed to check device partitions %s: %v", device.Name, err)
+			continue
+		}
+
+		// check if there is a file system on the device
+		fs, err := sys.GetDeviceFilesystems(device.Name, context.Executor)
+		if err != nil {
+			logger.Infof("failed to check device filesystem %s: %v", device.Name, err)
+			continue
+		}
+		device.Partitions = partitions
+		device.Filesystem = fs
+		devices = append(devices, *device)
+	}
+
+	logger.Infof("available devices: %+v", devices)
+	return devices, nil
+}

pkg/daemon/discover/discover_test.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package discover to discover unused devices.
+package discover
+
+import (
+	"testing"
+
+	"github.com/rook/rook/pkg/clusterd"
+	exectest "github.com/rook/rook/pkg/util/exec/test"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const udevOutput = `DEVLINKS=/dev/disk/by-id/scsi-36001405d27e5d898829468b90ce4ef8c /dev/disk/by-id/wwn-0x6001405d27e5d898829468b90ce4ef8c /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0 /dev/disk/by-uuid/f2d38cba-37da-411d-b7ba-9a6696c58174
+DEVNAME=/dev/sdk
+DEVPATH=/devices/platform/host6/session2/target6:0:0/6:0:0:0/block/sdk
+DEVTYPE=disk
+ID_BUS=scsi
+ID_FS_TYPE=ext2
+ID_FS_USAGE=filesystem
+ID_FS_UUID=f2d38cba-37da-411d-b7ba-9a6696c58174
+ID_FS_UUID_ENC=f2d38cba-37da-411d-b7ba-9a6696c58174
+ID_FS_VERSION=1.0
+ID_MODEL=disk01
+ID_MODEL_ENC=disk01\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20
+ID_PATH=ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0
+ID_PATH_TAG=ip-127_0_0_1_3260-iscsi-iqn_2016-06_world_srv_storage_target01-lun-0
+ID_REVISION=4.0
+ID_SCSI=1
+ID_SCSI_SERIAL=d27e5d89-8829-468b-90ce-4ef8c02f07fe
+ID_SERIAL=36001405d27e5d898829468b90ce4ef8c
+ID_SERIAL_SHORT=6001405d27e5d898829468b90ce4ef8c
+ID_TARGET_PORT=0
+ID_TYPE=disk
+ID_VENDOR=LIO-ORG
+ID_VENDOR_ENC=LIO-ORG\x20
+ID_WWN=0x6001405d27e5d898
+ID_WWN_VENDOR_EXTENSION=0x829468b90ce4ef8c
+ID_WWN_WITH_EXTENSION=0x6001405d27e5d898829468b90ce4ef8c
+MAJOR=8
+MINOR=160
+SUBSYSTEM=block
+TAGS=:systemd:
+USEC_INITIALIZED=15981915740802
+`
+
+func TestProbeDevices(t *testing.T) {
+	// set up mock execute so we can verify the partitioning happens on sda
+	executor := &exectest.MockExecutor{}
+	executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) {
+		logger.Infof("RUN Command for '%s'. %s arg %+v", name, command, args)
+		output := ""
+		switch name {
+		case "lsblk all":
+			output = "testa"
+		case "lsblk /dev/testa":
+			output = `SIZE="249510756352" ROTA="1" RO="0" TYPE="disk" PKNAME=""`
+		case "get filesystem type for testa":
+			output = udevOutput
+		case "get parent for device testa":
+			output = `       testa
+testa    testa1
+testa    testa2
+testa2   centos_host13-root
+testa2   centos_host13-swap
+testa2   centos_host13-home
+`
+		case "get disk testa fs uuid":
+			output = udevOutput
+
+		case "get disk testa fs serial":
+			output = udevOutput
+		}
+
+		return output, nil
+	}
+
+	context := &clusterd.Context{Executor: executor}
+
+	devices, err := probeDevices(context)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(devices))
+	assert.Equal(t, "ext2", devices[0].Filesystem)
+
+}

pkg/operator/cluster/ceph/mon/util.go

@@ -311,7 +311,7 @@ func genSecret(executor exec.Executor, configDir, name string, args []string) (s
 }
 
 func extractKey(contents string) (string, error) {
-	secret := sys.Awk(sys.Grep(string(contents), "key"), 3)
+	secret := sys.Awk(sys.Grep(string(contents), "key"), 3, " ")
 	if secret == "" {
 		return "", fmt.Errorf("failed to parse secret")
 	}

pkg/operator/cluster/ceph/osd/osd.go

@@ -28,6 +28,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/cluster/ceph/osd/config"
+	"github.com/rook/rook/pkg/operator/discover"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/util/display"
 	"k8s.io/api/core/v1"
@@ -105,7 +106,7 @@ func (c *Cluster) Start() error {
 	logger.Infof("start running osds in namespace %s", c.Namespace)
 
 	// create the artifacts for the osd to work with RBAC enabled
-	err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules, c.ownerRef)
+	err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules, &c.ownerRef)
 	if err != nil {
 		logger.Warningf("failed to init RBAC for OSDs. %+v", err)
 	}
@@ -168,9 +169,16 @@ func (c *Cluster) Start() error {
 			errorMessages = append(errorMessages, fmt.Sprintf("failed to set orchestration starting status for node %s: %+v", n.Name, err))
 			continue
 		}
-
+		devicesToUse := n.Devices
+		availDev, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace, n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices())
+		if deviceErr != nil {
+			logger.Warningf("failed to get devices for node %s cluster %s: %v", n.Name, c.Namespace, deviceErr)
+		} else {
+			devicesToUse = availDev
+			logger.Infof("avail devices for node %s: %+v", n.Name, availDev)
+		}
 		// create the replicaSet that will run the OSDs for this node
-		rs := c.makeReplicaSet(n.Name, n.Devices, n.Selection, resources, n.Config)
+		rs := c.makeReplicaSet(n.Name, devicesToUse, n.Selection, resources, n.Config)
 		_, err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Create(rs)
 		if err != nil {
 			if !errors.IsAlreadyExists(err) {

pkg/operator/discover/discover.go

@@ -0,0 +1,264 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package discover to discover devices on storage nodes.
+package discover
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+
+	"github.com/coreos/pkg/capnslog"
+	rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha1"
+	"github.com/rook/rook/pkg/clusterd"
+	discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	"github.com/rook/rook/pkg/util/sys"
+
+	"k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
+	"k8s.io/api/rbac/v1beta1"
+	kserrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+const (
+	discoverDaemonsetName             = "rook-discover"
+	discoverDaemonsetTolerationEnv    = "DISCOVER_TOLERATION"
+	discoverDaemonsetTolerationKeyEnv = "DISCOVER_TOLERATION_KEY"
+)
+
+var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-discover")
+
+var accessRules = []v1beta1.PolicyRule{
+	{
+		APIGroups: []string{""},
+		Resources: []string{"configmaps"},
+		Verbs:     []string{"get", "list", "update", "create", "delete"},
+	},
+}
+
+// Discover reference to be deployed
+type Discover struct {
+	clientset kubernetes.Interface
+}
+
+// New creates an instance of Discover
+func New(clientset kubernetes.Interface) *Discover {
+	return &Discover{
+		clientset: clientset,
+	}
+}
+
+// Start the discover
+func (d *Discover) Start(namespace, discoverImage string) error {
+
+	err := k8sutil.MakeRole(d.clientset, namespace, discoverDaemonsetName, accessRules, nil)
+	if err != nil {
+		return fmt.Errorf("failed to init RBAC for rook-discover. %+v", err)
+	}
+
+	err = d.createDiscoverDaemonSet(namespace, discoverImage)
+	if err != nil {
+		return fmt.Errorf("Error starting discover daemonset: %v", err)
+	}
+	return nil
+}
+
+func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage string) error {
+	privileged := false
+	ds := &extensions.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: discoverDaemonsetName,
+		},
+		Spec: extensions.DaemonSetSpec{
+			UpdateStrategy: extensions.DaemonSetUpdateStrategy{
+				Type: extensions.RollingUpdateDaemonSetStrategyType,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app": discoverDaemonsetName,
+					},
+				},
+				Spec: v1.PodSpec{
+					ServiceAccountName: discoverDaemonsetName,
+					Containers: []v1.Container{
+						{
+							Name:  discoverDaemonsetName,
+							Image: discoverImage,
+							Args:  []string{"discover"},
+							SecurityContext: &v1.SecurityContext{
+								Privileged: &privileged,
+							},
+							VolumeMounts: []v1.VolumeMount{
+								{
+									Name:      "dev",
+									MountPath: "/dev",
+									ReadOnly:  true,
+								},
+								{
+									Name:      "sys",
+									MountPath: "/sys",
+									ReadOnly:  true,
+								},
+								{
+									Name:      "udev",
+									MountPath: "/run/udev",
+									ReadOnly:  true,
+								},
+							},
+							Env: []v1.EnvVar{
+								k8sutil.NamespaceEnvVar(),
+								k8sutil.NodeEnvVar(),
+							},
+						},
+					},
+					Volumes: []v1.Volume{
+						{
+							Name: "dev",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/dev",
+								},
+							},
+						},
+						{
+							Name: "sys",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/sys",
+								},
+							},
+						},
+						{
+							Name: "udev",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/run/udev",
+								},
+							},
+						},
+					},
+					HostNetwork: false,
+				},
+			},
+		},
+	}
+
+	// Add toleration if any
+	tolerationValue := os.Getenv(discoverDaemonsetTolerationEnv)
+	if tolerationValue != "" {
+		ds.Spec.Template.Spec.Tolerations = []v1.Toleration{
+			{
+				Effect:   v1.TaintEffect(tolerationValue),
+				Operator: v1.TolerationOpExists,
+				Key:      os.Getenv(discoverDaemonsetTolerationKeyEnv),
+			},
+		}
+	}
+
+	_, err := d.clientset.Extensions().DaemonSets(namespace).Create(ds)
+	if err != nil {
+		if !kserrors.IsAlreadyExists(err) {
+			return fmt.Errorf("failed to create rook-discover daemon set. %+v", err)
+		}
+		logger.Infof("rook-discover daemonset already exists")
+	} else {
+		logger.Infof("rook-discover daemonset started")
+	}
+	return nil
+
+}
+
+func ListDevices(context *clusterd.Context, namespace, nodeName string) (map[string][]sys.LocalDisk, error) {
+	var devices map[string][]sys.LocalDisk
+	listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, discoverDaemon.AppName)}
+	cms, err := context.Clientset.CoreV1().ConfigMaps(namespace).List(listOpts)
+	if err != nil {
+		return devices, fmt.Errorf("failed to list device configmaps: %+v", err)
+	}
+	devices = make(map[string][]sys.LocalDisk, len(cms.Items))
+	for _, cm := range cms.Items {
+		node := cm.ObjectMeta.Labels[discoverDaemon.NodeAttr]
+		if len(nodeName) > 0 && node != nodeName {
+			continue
+		}
+		deviceJson := cm.Data[discoverDaemon.LocalDiskCMData]
+		logger.Debugf("node %s, device %s", node, deviceJson)
+
+		if len(node) == 0 || len(deviceJson) == 0 {
+			continue
+		}
+		var d []sys.LocalDisk
+		err = json.Unmarshal([]byte(deviceJson), &d)
+		if err != nil {
+			logger.Warningf("failed to unmarshal %s", deviceJson)
+			continue
+		}
+		devices[node] = d
+	}
+	logger.Debugf("devices %+v", devices)
+	return devices, nil
+}
+
+func GetAvailableDevices(context *clusterd.Context, nodeName, clusterName string, devices []rookalpha.Device, filter string, useAllDevices bool) ([]rookalpha.Device, error) {
+	results := []rookalpha.Device{}
+	if len(devices) == 0 && len(filter) == 0 && !useAllDevices {
+		return results, nil
+	}
+	namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
+	allDevices, err := ListDevices(context, namespace, nodeName)
+	if err != nil {
+		return results, err
+	}
+	nodeDevices, ok := allDevices[nodeName]
+	if !ok {
+		return results, fmt.Errorf("node %s has no devices", nodeName)
+	}
+	if len(devices) > 0 {
+		for i := range devices {
+			for j := range nodeDevices {
+				if devices[i].Name == nodeDevices[j].Name {
+					results = append(results, devices[i])
+				}
+			}
+		}
+	} else if len(filter) >= 0 {
+		for i := range nodeDevices {
+			//TODO support filter based on other keys
+			matched, err := regexp.Match(filter, []byte(nodeDevices[i].Name))
+			if err == nil && matched {
+				d := rookalpha.Device{
+					Name: nodeDevices[i].Name,
+				}
+				results = append(results, d)
+			}
+		}
+	} else if useAllDevices {
+		for i := range nodeDevices {
+			d := rookalpha.Device{
+				Name: nodeDevices[i].Name,
+			}
+			results = append(results, d)
+		}
+	}
+
+	return results, nil
+}

pkg/operator/discover/discover_test.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package discover to discover devices on storage nodes.
+package discover
+
+import (
+	"os"
+	"testing"
+
+	rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha1"
+	"github.com/rook/rook/pkg/clusterd"
+	discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	"github.com/rook/rook/pkg/operator/test"
+
+	"github.com/stretchr/testify/assert"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestStartDiscoveryDaemonset(t *testing.T) {
+	clientset := test.New(3)
+
+	os.Setenv(k8sutil.PodNamespaceEnvVar, "rook-system")
+	defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
+
+	os.Setenv(k8sutil.PodNameEnvVar, "rook-operator")
+	defer os.Unsetenv(k8sutil.PodNameEnvVar)
+
+	namespace := "ns"
+	a := New(clientset)
+
+	// start a basic cluster
+	err := a.Start(namespace, "rook/rook:myversion")
+	assert.Nil(t, err)
+
+	// check clusters rbac roles
+	_, err = clientset.CoreV1().ServiceAccounts(namespace).Get("rook-discover", metav1.GetOptions{})
+	assert.Nil(t, err)
+
+	_, err = clientset.RbacV1beta1().Roles(namespace).Get("rook-discover", metav1.GetOptions{})
+	assert.Nil(t, err)
+
+	// check daemonset parameters
+	agentDS, err := clientset.Extensions().DaemonSets(namespace).Get("rook-discover", metav1.GetOptions{})
+	assert.Nil(t, err)
+	assert.Equal(t, namespace, agentDS.Namespace)
+	assert.Equal(t, "rook-discover", agentDS.Name)
+	assert.False(t, *agentDS.Spec.Template.Spec.Containers[0].SecurityContext.Privileged)
+	volumes := agentDS.Spec.Template.Spec.Volumes
+	assert.Equal(t, 3, len(volumes))
+	volumeMounts := agentDS.Spec.Template.Spec.Containers[0].VolumeMounts
+	assert.Equal(t, 3, len(volumeMounts))
+	envs := agentDS.Spec.Template.Spec.Containers[0].Env
+	assert.Equal(t, 2, len(envs))
+	image := agentDS.Spec.Template.Spec.Containers[0].Image
+	assert.Equal(t, "rook/rook:myversion", image)
+	assert.Nil(t, agentDS.Spec.Template.Spec.Tolerations)
+}
+
+func TestGetAvailableDevices(t *testing.T) {
+	clientset := test.New(3)
+
+	ns := "rook-system"
+	nodeName := "node123"
+	os.Setenv(k8sutil.PodNamespaceEnvVar, ns)
+	defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
+
+	os.Setenv(k8sutil.PodNameEnvVar, "rook-operator")
+	defer os.Unsetenv(k8sutil.PodNameEnvVar)
+
+	data := make(map[string]string, 1)
+	data[discoverDaemon.LocalDiskCMData] = `[{"name":"sdd","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-36001405f826bd553d8c4dbf9f41c18be    /dev/disk/by-id/wwn-0x6001405f826bd553d8c4dbf9f41c18be /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-1","size":10737418240,"uuid":"","serial":"36001405f826bd553d8c4dbf9f41c18be","type":"disk","rotational":true,"readOnly":false,"ownPartition":true,"filesystem":"","vendor":"LIO-ORG","model":"disk02","wwn":"0x6001405f826bd553","wwnVendorExtension":"0x6001405f826bd553d8c4dbf9f41c18be","empty":true},{"name":"sdb","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-3600140577f462d9908b409d94114e042   /dev/disk/by-id/wwn-0x600140577f462d9908b409d94114e042 /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-3","size":5368709120,"uuid":"","serial":"3600140577f462d9908b409d94114e042","type":"disk","rotational":true,"readOnly":false,"ownPartition":false,"filesystem":"","vendor":"LIO-ORG","model":"disk04","wwn":"0x600140577f462d99","wwnVendorExtension":"0x600140577f462d9908b409d94114e042","empty":true},{"name":"sdc","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-3600140568c0bd28d4ee43769387c9f02    /dev/disk/by-id/wwn-0x600140568c0bd28d4ee43769387c9f02 /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-2","size":5368709120,"uuid":"","serial":"3600140568c0bd28d4ee43769387c9f02","type":"disk","rotational":true,"readOnly":false,"ownPartition":true,"filesystem":"","vendor":"LIO-ORG","model":"disk03","wwn":"0x600140568c0bd28d","wwnVendorExtension":"0x600140568c0bd28d4ee43769387c9f02","empty":true},{"name":"sda","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-36001405fc00c75fb4c243aa9d61987bd    /dev/disk/by-id/wwn-0x6001405fc00c75fb4c243aa9d61987bd /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0","size":10737418240,"uuid":"","serial":"36001405fc00c75fb4c243aa9d61987bd","type":"disk","rotational":true,"readOnly":false,"ownPartition":false,"filesystem":"","vendor":"LIO-ORG","model":"disk01","wwn":"0x6001405fc00c75fb","wwnVendorExtension":"0x6001405fc00c75fb4c243aa9d61987bd","empty":true},{"name":"nvme0n1","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/nvme-eui.002538c5710091a7","size":512110190592,"uuid":"","serial":"","type":"disk","rotational":false,"readOnly":false,"ownPartition":false,"filesystem":"","vendor":"","model":"","wwn":"","wwnVendorExtension":"","empty":true}]`
+	cm := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "local-device-" + nodeName,
+			Namespace: ns,
+			Labels: map[string]string{
+				k8sutil.AppAttr:         discoverDaemon.AppName,
+				discoverDaemon.NodeAttr: nodeName,
+			},
+		},
+		Data: data,
+	}
+	_, err := clientset.CoreV1().ConfigMaps(ns).Create(cm)
+	assert.Nil(t, err)
+	context := &clusterd.Context{
+		Clientset: clientset,
+	}
+	d := []rookalpha.Device{
+		{
+			Name: "sdc",
+		},
+		{
+			Name: "foo",
+		},
+	}
+
+	nodeDevices, err := ListDevices(context, ns, "" /* all nodes */)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(nodeDevices))
+
+	devices, err := GetAvailableDevices(context, nodeName, ns, d, "^sd.", false)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(devices))
+	devices, err = GetAvailableDevices(context, nodeName, ns, nil, "^sd.", false)
+	assert.Nil(t, err)
+	assert.Equal(t, 4, len(devices))
+	devices, err = GetAvailableDevices(context, nodeName, ns, nil, "", true)
+	assert.Nil(t, err)
+	assert.Equal(t, 5, len(devices))
+}

pkg/operator/k8sutil/rbac.go

@@ -32,9 +32,9 @@ const (
 	enableRBACEnv = "RBAC_ENABLED"
 )
 
-func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef metav1.OwnerReference) error {
+func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error {
 
-	err := makeServiceAccount(clientset, namespace, name, &ownerRef)
+	err := makeServiceAccount(clientset, namespace, name, ownerRef)
 	if err != nil {
 		return err
 	}
@@ -48,11 +48,13 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1
 	// the create will fail with an error that we're changing the permissions.
 	role := &v1beta1.Role{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:            name,
-			OwnerReferences: []metav1.OwnerReference{ownerRef},
+			Name: name,
 		},
 		Rules: rules,
 	}
+	if ownerRef != nil {
+		role.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef}
+	}
 	_, err = clientset.RbacV1beta1().Roles(namespace).Get(role.Name, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
 		logger.Infof("creating role %s in namespace %s", name, namespace)
@@ -67,8 +69,7 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1
 
 	binding := &v1beta1.RoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:            name,
-			OwnerReferences: []metav1.OwnerReference{ownerRef},
+			Name: name,
 		},
 		RoleRef: v1beta1.RoleRef{
 			Name:     name,
@@ -79,6 +80,9 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1
 			{Kind: "ServiceAccount", Name: name, Namespace: namespace},
 		},
 	}
+	if ownerRef != nil {
+		binding.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef}
+	}
 	_, err = clientset.RbacV1beta1().RoleBindings(namespace).Create(binding)
 	if err != nil && !errors.IsAlreadyExists(err) {
 		return fmt.Errorf("failed to create %s role binding in namespace %s. %+v", name, namespace, err)

pkg/operator/k8sutil/rbac_test.go

@@ -56,7 +56,8 @@ func TestMakeRole(t *testing.T) {
 		},
 	}
 
-	err := MakeRole(clientset, namespace, name, rules, metav1.OwnerReference{})
+	ownerRef := metav1.OwnerReference{}
+	err := MakeRole(clientset, namespace, name, rules, &ownerRef)
 
 	role, err := clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
 	assert.Nil(t, err)
@@ -82,7 +83,7 @@ func TestMakeRole(t *testing.T) {
 		},
 	}
 
-	err = MakeRole(clientset, namespace, name, newRules, metav1.OwnerReference{})
+	err = MakeRole(clientset, namespace, name, newRules, &ownerRef)
 	assert.Nil(t, err)
 	role, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
 	assert.Nil(t, err)
@@ -175,8 +176,8 @@ func TestMakeRoleRBACDisabled(t *testing.T) {
 			Verbs:     []string{"get", "list"},
 		},
 	}
-
-	err := MakeRole(clientset, namespace, name, rules, metav1.OwnerReference{})
+	ownerRef := metav1.OwnerReference{}
+	err := MakeRole(clientset, namespace, name, rules, &ownerRef)
 
 	_, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
 	assert.NotNil(t, err)

pkg/operator/operator.go

@@ -29,6 +29,7 @@ import (
 	"github.com/rook/rook/pkg/daemon/agent/flexvolume/attachment"
 	"github.com/rook/rook/pkg/operator/agent"
 	"github.com/rook/rook/pkg/operator/cluster"
+	"github.com/rook/rook/pkg/operator/discover"
 	"github.com/rook/rook/pkg/operator/file"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/operator/object"
@@ -86,6 +87,11 @@ func (o *Operator) Run() error {
 		return fmt.Errorf("Error starting agent daemonset: %v", err)
 	}
 
+	rookDiscover := discover.New(o.context.Clientset)
+	if err := rookDiscover.Start(namespace, o.rookImage); err != nil {
+		return fmt.Errorf("Error starting device discovery daemonset: %v", err)
+	}
+
 	signalChan := make(chan os.Signal, 1)
 	stopChan := make(chan struct{})
 	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

pkg/util/sys/device.go

@@ -19,9 +19,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
-
 	"strconv"
+	"strings"
 
 	"github.com/google/uuid"
 	"github.com/rook/rook/pkg/util/exec"
@@ -37,9 +36,48 @@ const (
 )
 
 type Partition struct {
-	Name  string
-	Size  uint64
-	Label string
+	Name       string
+	Size       uint64
+	Label      string
+	Filesystem string
+}
+
+// LocalDevice contains information about an unformatted block device
+type LocalDisk struct {
+	// Name is the device name
+	Name string `json:"name"`
+	// Parent is the device parent's name
+	Parent string `json:"parent"`
+	// HasChildren is whether the device has a children device
+	HasChildren bool `json:"hasChildren"`
+	// DevLinks is the persistent device path on the host
+	DevLinks string `json:"devLinks"`
+	// Size is the device capacity in byte
+	Size uint64 `json:"size"`
+	// UUID is used by /dev/disk/by-uuid
+	UUID string `json:"uuid"`
+	// Serial is the disk serial used by /dev/disk/by-id
+	Serial string `json:"serial"`
+	// Type is disk type
+	Type string `json:"type"`
+	// Rotational is the boolean whether the device is rotational: true for hdd, false for ssd and nvme
+	Rotational bool `json:"rotational"`
+	// ReadOnly is the boolean whether the device is readonly
+	Readonly bool `json:"readOnly"`
+	// Partitions is a partition slice
+	Partitions []Partition
+	// Filesystem is the filesystem currently on the device
+	Filesystem string `json:"filesystem"`
+	// Vendor is the device vendor
+	Vendor string `json:"vendor"`
+	// Model is the device model
+	Model string `json:"model"`
+	// WWN is the world wide name of the device
+	WWN string `json:"wwn"`
+	// WWNVendorExtension is the WWN_VENDOR_EXTENSION from udev info
+	WWNVendorExtension string `json:"wwnVendorExtension"`
+	// Empty checks whether the device is completely empty
+	Empty bool `json:"empty"`
 }
 
 func ListDevices(executor exec.Executor) ([]string, error) {
@@ -52,7 +90,7 @@ func ListDevices(executor exec.Executor) ([]string, error) {
 	return strings.Split(devices, "\n"), nil
 }
 
-func GetDevicePartitions(device string, executor exec.Executor) (partitions []*Partition, unusedSpace uint64, err error) {
+func GetDevicePartitions(device string, executor exec.Executor) (partitions []Partition, unusedSpace uint64, err error) {
 	cmd := fmt.Sprintf("lsblk /dev/%s", device)
 	output, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk", fmt.Sprintf("/dev/%s", device),
 		"--bytes", "--pairs", "--output", "NAME,SIZE,TYPE,PKNAME")
@@ -74,7 +112,7 @@ func GetDevicePartitions(device string, executor exec.Executor) (partitions []*P
 			}
 		} else if props["PKNAME"] == device && props["TYPE"] == PartType {
 			// found a partition
-			p := &Partition{Name: name}
+			p := Partition{Name: name}
 			p.Size, err = strconv.ParseUint(props["SIZE"], 10, 64)
 			if err != nil {
 				return nil, 0, fmt.Errorf("failed to get partition %s size. %+v", name, err)
@@ -87,6 +125,12 @@ func GetDevicePartitions(device string, executor exec.Executor) (partitions []*P
 			}
 			p.Label = label
 
+			fs, err := GetDeviceFilesystems(name, executor)
+			if err != nil {
+				return nil, 0, err
+			}
+			p.Filesystem = fs
+
 			partitions = append(partitions, p)
 		}
 	}
@@ -120,15 +164,25 @@ func GetDevicePropertiesFromPath(devicePath string, executor exec.Executor) (map
 	return parseKeyValuePairString(output), nil
 }
 
+func GetUdevInfo(device string, executor exec.Executor) (map[string]string, error) {
+	cmd := fmt.Sprintf("udevadm info %s", device)
+	output, err := executor.ExecuteCommandWithOutput(false, cmd, "udevadm", "info", "--query=property", fmt.Sprintf("/dev/%s", device))
+	if err != nil {
+		return nil, err
+	}
+
+	return parseUdevInfo(output), nil
+}
+
 // get the file systems availab
 func GetDeviceFilesystems(device string, executor exec.Executor) (string, error) {
 	cmd := fmt.Sprintf("get filesystem type for %s", device)
-	output, err := executor.ExecuteCommandWithOutput(false, cmd, "df", "--output=source,fstype")
+	output, err := executor.ExecuteCommandWithOutput(false, cmd, "udevadm", "info", "--query=property", fmt.Sprintf("/dev/%s", device))
 	if err != nil {
 		return "", fmt.Errorf("command %s failed: %+v", cmd, err)
 	}
 
-	return parseDFOutput(device, output), nil
+	return parseFS(output), nil
 }
 
 func RemovePartitions(device string, executor exec.Executor) error {
@@ -177,13 +231,14 @@ func GetPartitionLabel(deviceName string, executor exec.Executor) (string, error
 	// look up the partition's label with blkid because lsblk relies on udev which is
 	// not available in containers
 	devicePath := fmt.Sprintf("/dev/%s", deviceName)
-	cmd := fmt.Sprintf("blkid %s", devicePath)
-	output, err := executor.ExecuteCommandWithOutput(false, cmd, "blkid", devicePath, "-s", "PARTLABEL", "-o", "value")
+	cmd := fmt.Sprintf("udevadm %s", devicePath)
+	output, err := executor.ExecuteCommandWithOutput(false, cmd,
+		"udevadm", "info", "--query=property", devicePath)
 	if err != nil {
 		return "", fmt.Errorf("failed to get partition label for device %s: %+v", deviceName, err)
 	}
 
-	return output, nil
+	return parsePartLabel(output), nil
 }
 
 // look up the mount point of the given device.  empty string returned if device is not mounted.
@@ -195,7 +250,7 @@ func GetDeviceMountPoint(deviceName string, executor exec.Executor) (string, err
 	}
 
 	searchFor := fmt.Sprintf("^/dev/%s on", deviceName)
-	mountPoint := Awk(Grep(output, searchFor), 3)
+	mountPoint := Awk(Grep(output, searchFor), 3, " ")
 	return mountPoint, nil
 }
 
@@ -208,7 +263,7 @@ func GetDeviceFromMountPoint(mountPoint string, executor exec.Executor) (string,
 	}
 
 	searchFor := fmt.Sprintf("on %s ", mountPoint)
-	device := Awk(Grep(output, searchFor), 1)
+	device := Awk(Grep(output, searchFor), 1, " ")
 	return device, nil
 }
 
@@ -254,31 +309,46 @@ func UnmountDevice(devicePath string, executor exec.Executor) error {
 	return nil
 }
 
-func DoesDeviceHaveChildren(device string, executor exec.Executor) (bool, error) {
-	cmd := fmt.Sprintf("check children for device %s", device)
-	output, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk --all -n -l --output PKNAME")
+func GetFSUUID(device string, executor exec.Executor) (string, error) {
+	cmd := fmt.Sprintf("get disk %s fs uuid", device)
+	output, err := executor.ExecuteCommandWithOutput(false, cmd,
+		"udevadm", "info", "--query=property", fmt.Sprintf("/dev/%s", device))
 	if err != nil {
-		return false, fmt.Errorf("command %s failed: %+v", cmd, err)
+		return "", err
 	}
-
-	searchFor := fmt.Sprintf("^%s$", device)
-	children := Grep(output, searchFor)
-
-	return children != "", nil
+	return parseFSUUID(output), nil
 }
 
-// finds the file system(s) for the device in the output of 'df'
-func parseDFOutput(device, output string) string {
-	var fs []string
-	lines := strings.Split(output, "\n")
-	for _, line := range lines {
-		if strings.HasPrefix(line, fmt.Sprintf("/dev/%s", device)) {
-			words := strings.Split(line, " ")
-			fs = append(fs, words[len(words)-1])
+func CheckIfDeviceAvailable(executor exec.Executor, name string) (bool, string, error) {
+	ownPartitions := true
+	partitions, _, err := GetDevicePartitions(name, executor)
+	if err != nil {
+		return false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err)
+	}
+	if !RookOwnsPartitions(partitions) {
+		ownPartitions = false
+	}
+
+	// check if there is a file system on the device
+	devFS, err := GetDeviceFilesystems(name, executor)
+	if err != nil {
+		return false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err)
+	}
+
+	return ownPartitions, devFS, nil
+}
+
+func RookOwnsPartitions(partitions []Partition) bool {
+
+	// if there are partitions, they must all have the rook osd label
+	for _, p := range partitions {
+		if !strings.HasPrefix(p.Label, "ROOK-OSD") {
+			return false
 		}
 	}
 
-	return strings.Join(fs, ",")
+	// if there are no partitions, or the partitions are all from rook OSDs, then rook owns the device
+	return true
 }
 
 // finds the disk uuid in the output of sgdisk
@@ -323,3 +393,42 @@ func parseKeyValuePairString(propsRaw string) map[string]string {
 
 	return propMap
 }
+
+// find fs from udevadm info
+func parseFS(output string) string {
+	m := parseUdevInfo(output)
+	if v, ok := m["ID_FS_TYPE"]; ok {
+		return v
+	}
+	return ""
+}
+
+// find fs from udevadm info
+func parseFSUUID(output string) string {
+	m := parseUdevInfo(output)
+	if v, ok := m["ID_FS_UUID"]; ok {
+		return v
+	}
+	return ""
+}
+
+// find partition label from udevadm info
+func parsePartLabel(output string) string {
+	m := parseUdevInfo(output)
+	if v, ok := m["ID_PART_ENTRY_NAME"]; ok {
+		return v
+	}
+	return ""
+}
+
+func parseUdevInfo(output string) map[string]string {
+	lines := strings.Split(output, "\n")
+	result := make(map[string]string, len(lines))
+	for _, v := range lines {
+		pairs := strings.Split(v, "=")
+		if len(pairs) > 1 {
+			result[pairs[0]] = pairs[1]
+		}
+	}
+	return result
+}

pkg/util/sys/device_test.go

@@ -24,6 +24,68 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const (
+	udevOutput = `DEVLINKS=/dev/disk/by-id/scsi-36001405d27e5d898829468b90ce4ef8c /dev/disk/by-id/wwn-0x6001405d27e5d898829468b90ce4ef8c /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0 /dev/disk/by-uuid/f2d38cba-37da-411d-b7ba-9a6696c58174
+DEVNAME=/dev/sdk
+DEVPATH=/devices/platform/host6/session2/target6:0:0/6:0:0:0/block/sdk
+DEVTYPE=disk
+ID_BUS=scsi
+ID_FS_TYPE=ext2
+ID_FS_USAGE=filesystem
+ID_FS_UUID=f2d38cba-37da-411d-b7ba-9a6696c58174
+ID_FS_UUID_ENC=f2d38cba-37da-411d-b7ba-9a6696c58174
+ID_FS_VERSION=1.0
+ID_MODEL=disk01
+ID_MODEL_ENC=disk01\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20
+ID_PATH=ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0
+ID_PATH_TAG=ip-127_0_0_1_3260-iscsi-iqn_2016-06_world_srv_storage_target01-lun-0
+ID_REVISION=4.0
+ID_SCSI=1
+ID_SCSI_SERIAL=d27e5d89-8829-468b-90ce-4ef8c02f07fe
+ID_SERIAL=36001405d27e5d898829468b90ce4ef8c
+ID_SERIAL_SHORT=6001405d27e5d898829468b90ce4ef8c
+ID_TARGET_PORT=0
+ID_TYPE=disk
+ID_VENDOR=LIO-ORG
+ID_VENDOR_ENC=LIO-ORG\x20
+ID_WWN=0x6001405d27e5d898
+ID_WWN_VENDOR_EXTENSION=0x829468b90ce4ef8c
+ID_WWN_WITH_EXTENSION=0x6001405d27e5d898829468b90ce4ef8c
+MAJOR=8
+MINOR=160
+SUBSYSTEM=block
+TAGS=:systemd:
+USEC_INITIALIZED=15981915740802
+`
+	udevPartOutput = `ID_PART_ENTRY_DISK=8:32
+ID_PART_ENTRY_NAME=%s
+ID_PART_ENTRY_NUMBER=3
+ID_PART_ENTRY_OFFSET=3278848
+ID_PART_ENTRY_SCHEME=gpt
+ID_PART_ENTRY_SIZE=7206879
+ID_PART_ENTRY_TYPE=0fc63daf-8483-4772-8e79-3d69d8477de4
+ID_PART_ENTRY_UUID=2089640e-bdeb-4fb4-aaec-88e165780b88
+ID_PART_TABLE_TYPE=gpt
+ID_PART_TABLE_UUID=46242f96-6cf7-4e5d-b4bd-9d046e6ad920
+ID_REVISION=4.0
+ID_SCSI=1
+ID_SCSI_SERIAL=68c0bd28-d4ee-4376-9387-c9f02c53b3f2
+ID_SERIAL=3600140568c0bd28d4ee43769387c9f02
+ID_SERIAL_SHORT=600140568c0bd28d4ee43769387c9f02
+ID_TARGET_PORT=0
+ID_TYPE=disk
+ID_VENDOR=LIO-ORG
+ID_VENDOR_ENC=LIO-ORG\x20
+ID_WWN=0x600140568c0bd28d
+ID_WWN_VENDOR_EXTENSION=0x4ee43769387c9f02
+ID_WWN_WITH_EXTENSION=0x600140568c0bd28d4ee43769387c9f02
+MAJOR=8
+MINOR=35
+PARTN=3
+SUBSYSTEM=block
+`
+)
+
 func TestFindUUID(t *testing.T) {
 	output := `Disk /dev/sdb: 10485760 sectors, 5.0 GiB
 Logical sector size: 512 bytes
@@ -39,24 +101,10 @@ Total free space is 20971453 sectors (10.0 GiB)
 }
 
 func TestParseFileSystem(t *testing.T) {
-	output := `Filesystem     Type
-devtmpfs       devtmpfs
-/dev/sda9      ext4
-/dev/sda3      ext4
-/dev/sda1      vfat
-tmpfs          tmpfs
-tmpfs          tmpfs
-/dev/sda6      ext4
-sdc            tmpfs`
+	output := udevOutput
 
-	result := parseDFOutput("sda", output)
-	assert.Equal(t, "ext4,ext4,vfat,ext4", result)
-
-	result = parseDFOutput("sdb", output)
-	assert.Equal(t, "", result)
-
-	result = parseDFOutput("sdc", output)
-	assert.Equal(t, "", result)
+	result := parseFS(output)
+	assert.Equal(t, "ext2", result)
 }
 
 func TestGetDeviceFromMountPoint(t *testing.T) {
@@ -127,6 +175,7 @@ func TestGetPartitions(t *testing.T) {
 	executor := &exectest.MockExecutor{
 		MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, arg ...string) (string, error) {
 			run++
+			logger.Infof("run %d action %s", run, actionName)
 			switch {
 			case run == 1:
 				return `NAME="sdc" SIZE="100000" TYPE="disk" PKNAME=""`, nil
@@ -136,12 +185,15 @@ NAME="sdb2" SIZE="10" TYPE="part" PKNAME="sdb"
 NAME="sdb3" SIZE="20" TYPE="part" PKNAME="sdb"
 NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil
 			case run == 3:
-				return "ROOK-OSD0-DB", nil
-			case run == 4:
-				return "ROOK-OSD0-BLOCK", nil
+				return fmt.Sprintf(udevPartOutput, "ROOK-OSD0-DB"), nil
+			case run == 4 || run == 6 || run == 8:
+				// get filesystem
+				return "", nil
 			case run == 5:
-				return "ROOK-OSD0-WAL", nil
-			case run == 6:
+				return fmt.Sprintf(udevPartOutput, "ROOK-OSD0-BLOCK"), nil
+			case run == 7:
+				return fmt.Sprintf(udevPartOutput, "ROOK-OSD0-WAL"), nil
+			case run == 9:
 				return `NAME="sda" SIZE="19818086400" TYPE="disk" PKNAME=""
 NAME="sda4" SIZE="1073741824" TYPE="part" PKNAME="sda"
 NAME="sda2" SIZE="2097152" TYPE="part" PKNAME="sda"
@@ -151,22 +203,14 @@ NAME="sda3" SIZE="1073741824" TYPE="part" PKNAME="sda"
 NAME="usr" SIZE="1065345024" TYPE="crypt" PKNAME="sda3"
 NAME="sda1" SIZE="134217728" TYPE="part" PKNAME="sda"
 NAME="sda6" SIZE="134217728" TYPE="part" PKNAME="sda"`, nil
-			case run == 7:
-				return "USR-B", nil
-			case run == 8:
-				return "BIOS-BOOT", nil
 			case run == 9:
-				return "ROOT", nil
+				return fmt.Sprintf(udevPartOutput, "ROOT"), nil
 			case run == 10:
-				return "OEM-CONFIG", nil
+				return fmt.Sprintf(udevPartOutput, "OEM-CONFIG"), nil
 			case run == 11:
-				return "USR-A", nil
+				return fmt.Sprintf(udevPartOutput, "USR-A"), nil
 			case run == 12:
-				return "EFI-SYSTEM", nil
-			case run == 13:
-				return "OEM", nil
-			case run == 14:
-				return "", nil
+				return fmt.Sprintf(udevPartOutput, "EFI-SYSTEM"), nil
 			}
 			return "", nil
 		},
@@ -194,3 +238,8 @@ NAME="sda6" SIZE="134217728" TYPE="part" PKNAME="sda"`, nil
 	assert.Nil(t, err)
 	assert.Equal(t, 0, len(partitions))
 }
+
+func TestParseUdevInfo(t *testing.T) {
+	m := parseUdevInfo(udevOutput)
+	assert.Equal(t, m["ID_FS_TYPE"], "ext2")
+}

pkg/util/sys/parse.go

@@ -37,13 +37,13 @@ func Grep(input, searchFor string) string {
 
 // awk finds the space-delimited token at the given position.
 // 0 returns the whole line, while 1 is the first token.
-func Awk(input string, position int) string {
+func Awk(input string, position int, delimit string) string {
 	logger.Debugf("awk %d from %s", position, input)
 	if position == 0 {
 		return input
 	}
 
-	words := strings.Split(strings.TrimSpace(input), " ")
+	words := strings.Split(strings.TrimSpace(input), delimit)
 	position--
 	if position < 0 || position >= len(words) {
 		// out of range

pkg/util/sys/parse_test.go

@@ -56,6 +56,6 @@ func TestAwk(t *testing.T) {
 }
 
 func testAwk(t *testing.T, index int, input, expected string) {
-	out := Awk(input, index)
+	out := Awk(input, index, " ")
 	assert.Equal(t, expected, out)
 }

tests/framework/utils/helm_helper.go

@@ -62,7 +62,7 @@ func (h *HelmHelper) GetLocalRookHelmChartVersion(chartName string) (string, err
 	}
 	cd := strings.Replace(sys.Grep(result, chartName), "\t", " ", 2)
 
-	return sys.Awk(cd, 2), nil
+	return sys.Awk(cd, 2, " "), nil
 }
 
 //InstallLocalRookHelmChart installs a give helm chart

tests/integration/base_deploy_test.go

@@ -45,6 +45,8 @@ func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNames
 		"Make sure there is 1 rook-operator present in Running state")
 	assert.True(s.T(), k8sh.CheckPodCountAndState("rook-agent", opNamespace, 1, "Running"),
 		"Make sure there is 1 rook-agent present in Running state")
+	assert.True(s.T(), k8sh.CheckPodCountAndState("rook-discover", opNamespace, 1, "Running"),
+		"Make sure there is 1 rook-discover present in Running state")
 	assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", clusterNamespace, 1, "Running"),
 		"Make sure there is 1 rook-ceph-mgr present in Running state")
 	assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-osd", clusterNamespace, 1, "Running"),

tests/integration/multi_cluster_deploy_test.go

@@ -160,6 +160,9 @@ func (o MCTestOperations) SetUp() {
 	require.True(o.T(), o.kh.IsPodInExpectedState("rook-agent", installer.SystemNamespace(o.namespace1), "Running"),
 		"Make sure rook-agent is in running state")
 
+	require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", installer.SystemNamespace(o.namespace1), "Running"),
+		"Make sure rook-discover is in running state")
+
 	time.Sleep(10 * time.Second)
 
 	// start the two clusters in parallel