diff --git a/pkg/cephmgr/osd/daemon.go b/pkg/cephmgr/osd/daemon.go index 8fa44c484..a63a28017 100644 --- a/pkg/cephmgr/osd/daemon.go +++ b/pkg/cephmgr/osd/daemon.go @@ -22,6 +22,7 @@ import ( "log" "os" "path" + "regexp" "time" "github.com/rook/rook/pkg/cephmgr/mon" @@ -107,6 +108,15 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L if fs == "" && ownPartitions { if desiredDevices == "all" { available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} + } else if desiredDevices != "" { + // the desired devices is a regular expression + matched, err := regexp.Match(desiredDevices, []byte(device.Name)) + if err == nil && matched { + available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} + } else { + logger.Infof("skipping device %s that does not match the regular expression %s. %+v", device.Name, desiredDevices, err) + } + } else { logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name) } diff --git a/pkg/cephmgr/osd/daemon_test.go b/pkg/cephmgr/osd/daemon_test.go index 93083e454..916890720 100644 --- a/pkg/cephmgr/osd/daemon_test.go +++ b/pkg/cephmgr/osd/daemon_test.go @@ -16,12 +16,17 @@ limitations under the License. package osd import ( + "fmt" + "strings" "testing" "os" "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/clusterd/inventory" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/rook/rook/pkg/util/proc" "github.com/stretchr/testify/assert" ) @@ -59,3 +64,71 @@ func TestStoreOSDDirMap(t *testing.T) { assert.Equal(t, 0, dirMap[context.ConfigDir]) assert.Equal(t, 23, dirMap["/tmp/mydir"]) } + +func TestAvailableDevices(t *testing.T) { + executor := &exectest.MockExecutor{} + // set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem + executor.MockExecuteCommandWithOutput = func(name string, command string, args ...string) (string, error) { + logger.Infof("OUTPUT for %s. %s %+v", name, command, args) + + if command == "lsblk" { + if strings.Index(name, "sdb") != -1 { + // /dev/sdb has a partition + return `NAME="sdb" SIZE="65" TYPE="disk" PKNAME="" PARTLABEL="" +NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="MY-PART"`, nil + } + return "", nil + } else if command == "df" { + if strings.Index(name, "sdc") != -1 { + // /dev/sdc has a file system + return "/dev/sdc ext4", nil + } + return "", nil + } + + return "", fmt.Errorf("unknown command %s", command) + } + + context := &clusterd.DaemonContext{ProcMan: proc.New(executor), Executor: executor} + devices := []*inventory.LocalDisk{ + &inventory.LocalDisk{Name: "sda"}, + &inventory.LocalDisk{Name: "sdb"}, + &inventory.LocalDisk{Name: "sdc"}, + &inventory.LocalDisk{Name: "sdd"}, + &inventory.LocalDisk{Name: "rda"}, + &inventory.LocalDisk{Name: "rdb"}, + } + // select all devices + mapping, err := getAvailableDevices(context, devices, "all") + assert.Nil(t, err) + assert.Equal(t, 4, len(mapping.Entries)) + assert.Equal(t, -1, mapping.Entries["sda"].Data) + assert.Equal(t, -1, mapping.Entries["sdd"].Data) + assert.Equal(t, -1, mapping.Entries["rda"].Data) + assert.Equal(t, -1, mapping.Entries["rdb"].Data) + + // select no devices + mapping, err = getAvailableDevices(context, devices, "") + assert.Nil(t, err) + assert.Equal(t, 0, len(mapping.Entries)) + + // select the sda devices + mapping, err = getAvailableDevices(context, devices, "^sd.$") + assert.Nil(t, err) + assert.Equal(t, 2, len(mapping.Entries)) + assert.Equal(t, -1, mapping.Entries["sda"].Data) + assert.Equal(t, -1, mapping.Entries["sdd"].Data) + + // select an exact device + mapping, err = getAvailableDevices(context, devices, "sdd") + assert.Nil(t, err) + assert.Equal(t, 1, len(mapping.Entries)) + assert.Equal(t, -1, mapping.Entries["sdd"].Data) + + // select all devices except those that have a prefix of "s" + mapping, err = getAvailableDevices(context, devices, "^[^s]") + assert.Nil(t, err) + assert.Equal(t, 2, len(mapping.Entries)) + assert.Equal(t, -1, mapping.Entries["rda"].Data) + assert.Equal(t, -1, mapping.Entries["rdb"].Data) +} diff --git a/pkg/operator/cluster.go b/pkg/operator/cluster.go index d70fa9f47..60a513099 100644 --- a/pkg/operator/cluster.go +++ b/pkg/operator/cluster.go @@ -56,6 +56,9 @@ type ClusterSpec struct { // Whether to consume all the storage devices found on a machine UseAllDevices bool `json:"useAllDevices"` + + // A regular expression to allow more fine-grained selection of devices on nodes across the cluster + DeviceFilter string `json:"deviceFilter"` } func newCluster(spec ClusterSpec, factory client.ConnectionFactory, clientset kubernetes.Interface) *Cluster { @@ -92,7 +95,7 @@ func (c *Cluster) CreateInstance() error { } // Start the OSDs - osds := osd.New(c.Spec.Namespace, c.Spec.Version, c.Spec.UseAllDevices) + osds := osd.New(c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.UseAllDevices) err = osds.Start(c.clientset, cluster) if err != nil { return fmt.Errorf("failed to start the osds. %+v", err) diff --git a/pkg/operator/osd/osd.go b/pkg/operator/osd/osd.go index 78984d8ad..d8adaa954 100644 --- a/pkg/operator/osd/osd.go +++ b/pkg/operator/osd/osd.go @@ -36,12 +36,14 @@ type Cluster struct { Keyring string Version string useAllDevices bool + deviceFilter string } -func New(namespace, version string, useAllDevices bool) *Cluster { +func New(namespace, version, deviceFilter string, useAllDevices bool) *Cluster { return &Cluster{ Namespace: namespace, Version: version, + deviceFilter: deviceFilter, useAllDevices: useAllDevices, } } @@ -100,7 +102,9 @@ func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) v1.Container { command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s ", k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name) - if c.useAllDevices { + if c.deviceFilter != "" { + command += fmt.Sprintf("--data-devices=%s ", c.deviceFilter) + } else if c.useAllDevices { command += fmt.Sprintf("--data-devices=all ") } diff --git a/pkg/operator/osd/osd_test.go b/pkg/operator/osd/osd_test.go index a41b55016..f7677927f 100644 --- a/pkg/operator/osd/osd_test.go +++ b/pkg/operator/osd/osd_test.go @@ -27,7 +27,7 @@ import ( ) func TestStartDaemonset(t *testing.T) { - c := New("ns", "myversion", false) + c := New("ns", "myversion", "", false) clientset := fake.NewSimpleClientset() @@ -52,7 +52,7 @@ func TestDaemonset(t *testing.T) { } func testPodDevices(t *testing.T, useDevices bool) { - c := New("ns", "myversion", useDevices) + c := New("ns", "myversion", "", useDevices) info := testop.CreateClusterInfo(1) daemonSet, err := c.makeDaemonSet(info) assert.Nil(t, err)