Merge pull request #404 from travisn/dir-store-default

Use a directory by default rather than taking over devices

cmd/rook-operator/main.go

@@ -47,6 +47,7 @@ type config struct {
 	cephConfigOverride string
 	clusterInfo        mon.ClusterInfo
 	monEndpoints       string
+	useAllDevices      bool
 }
 
 var logLevelRaw string
@@ -74,6 +75,8 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&cfg.dataDir, "data-dir", "/var/lib/rook", "directory for storing configuration")
 	rootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "INFO", "logging level for logging/tracing output (valid values: CRITICAL,ERROR,WARNING,NOTICE,INFO,DEBUG,TRACE)")
 
+	rootCmd.Flags().BoolVar(&cfg.useAllDevices, "use-all-devices", false, "true to use all storage devices, false to require local device selection")
+
 	// load the environment variables
 	flags.SetFlagsFromEnv(rootCmd.Flags(), "ROOK_OPERATOR")
 	flags.SetFlagsFromEnv(rootCmd.PersistentFlags(), "ROOK_OPERATOR")
@@ -96,7 +99,7 @@ func startOperator(cmd *cobra.Command, args []string) error {
 	}
 
 	logger.Infof("starting operator. containerVersion=%s", cfg.containerVersion)
-	op := operator.New(k8sutil.Namespace, cephd.New(), clientset, cfg.containerVersion)
+	op := operator.New(k8sutil.Namespace, cephd.New(), clientset, cfg.containerVersion, cfg.useAllDevices)
 	err = op.Run()
 	if err != nil {
 		fmt.Printf("failed to run operator. %+v\n", err)

cmd/rookd/osd.go

@@ -33,10 +33,14 @@ var osdCmd = &cobra.Command{
 	Hidden: true,
 }
 var (
-	osdCluster mon.ClusterInfo
+	osdCluster     mon.ClusterInfo
+	osdDataDevices string
 )
 
 func init() {
+	osdCmd.Flags().StringVar(&osdDataDevices, "data-devices", "", "the device names to use, or \"all\"")
+	flags.SetFlagsFromEnv(monCmd.Flags(), "ROOKD")
+
 	osdCmd.RunE = startOSD
 }
 
@@ -47,13 +51,12 @@ func startOSD(cmd *cobra.Command, args []string) error {
 
 	setLogLevel()
 
-	devices := ""
 	metadataDevice := ""
 	forceFormat := false
 	location := ""
 	bluestoreConfig := partition.BluestoreConfig{DatabaseSizeMB: 512} // FIX
 	clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
-	agent := osd.NewAgent(cephd.New(), devices, metadataDevice, forceFormat, location, bluestoreConfig, &clusterInfo)
+	agent := osd.NewAgent(cephd.New(), osdDataDevices, metadataDevice, forceFormat, location, bluestoreConfig, &clusterInfo)
 	context := clusterd.NewDaemonContext(cfg.dataDir, cfg.cephConfigOverride, cfg.logLevel)
 
 	return osd.Run(context, agent)

pkg/cephmgr/osd/agent.go

@@ -329,13 +329,16 @@ func (a *OsdAgent) configureDirs(context *clusterd.Context, dirs map[string]int)
 				return err
 			}
 
+			dirs[dirPath] = *osdID
 			config.id = *osdID
 			config.uuid = *osdUUID
 
 			// set the desired state of the dir with the osd id
-			err = associateOsdIDWithDevice(context.EtcdClient, context.NodeID, dirPath, config.id, config.dir)
-			if err != nil {
-				return fmt.Errorf("failed to associate osd id %d with the data dir", config.id)
+			if context.EtcdClient != nil {
+				err = associateOsdIDWithDevice(context.EtcdClient, context.NodeID, dirPath, config.id, config.dir)
+				if err != nil {
+					return fmt.Errorf("failed to associate osd id %d with the data dir", config.id)
+				}
 			}
 		}
 

pkg/cephmgr/osd/daemon.go

@@ -16,8 +16,12 @@ limitations under the License.
 package osd
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
+	"os"
+	"path"
 	"time"
 
 	"github.com/rook/rook/pkg/cephmgr/mon"
@@ -26,6 +30,10 @@ import (
 	"github.com/rook/rook/pkg/util/sys"
 )
 
+const (
+	dirOSDConfigFilename = "osd-dirs"
+)
+
 func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
 
 	logger.Infof("discovering hardware")
@@ -53,7 +61,7 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
 	}
 
 	// initialize the desired osds
-	devices, err := getAvailableDevices(dcontext, hardware.Disks)
+	devices, err := getAvailableDevices(dcontext, hardware.Disks, agent.devices)
 	if err != nil {
 		return fmt.Errorf("failed to get available devices. %+v", err)
 	}
@@ -64,13 +72,19 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
 		return fmt.Errorf("failed to configure devices. %+v", err)
 	}
 
-	if len(devices.Entries) == 0 {
-		dirs := map[string]int{context.ConfigDir: -1}
-		logger.Infof("configuring osd dir: %+v", dirs)
-		err = agent.configureDirs(context, dirs)
-		if err != nil {
-			return fmt.Errorf("failed to configure dir %s. %+v", context.ConfigDir, err)
-		}
+	// initialize the data directories
+	dirs, err := getDataDirs(dcontext, true)
+	if err != nil {
+		return fmt.Errorf("failed to get data dirs. %+v", err)
+	}
+	logger.Infof("configuring osd dirs: %+v", dirs)
+	err = agent.configureDirs(context, dirs)
+	if err != nil {
+		return fmt.Errorf("failed to configure dirs %v. %+v", dirs, err)
+	}
+	err = saveDirConfig(dcontext, dirs)
+	if err != nil {
+		return fmt.Errorf("failed to save osd dir config. %+v", err)
 	}
 
 	// FIX
@@ -79,7 +93,7 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
 	return nil
 }
 
-func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk) (*DeviceOsdMapping, error) {
+func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk, desiredDevices string) (*DeviceOsdMapping, error) {
 	available := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}}
 	for _, device := range devices {
 		if device.Type == sys.PartType {
@@ -90,9 +104,56 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L
 			return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err)
 		}
 		if fs == "" && ownPartitions {
-			available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
+			if desiredDevices == "all" {
+				available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
+			} else {
+				logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name)
+			}
 		}
 	}
 
 	return available, nil
 }
+
+func getDataDirs(context *clusterd.DaemonContext, useDataDir bool) (map[string]int, error) {
+	filePath := path.Join(context.ConfigDir, dirOSDConfigFilename)
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		// the config file doesn't exist yet, just return the empty map unless the data dir is desired
+		if useDataDir {
+			return map[string]int{context.ConfigDir: unassignedOSDID}, nil
+		}
+
+		return map[string]int{}, nil
+	}
+
+	b, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	var dirs map[string]int
+	err = json.Unmarshal(b, &dirs)
+	if err != nil {
+		return nil, err
+	}
+	return dirs, nil
+}
+
+func saveDirConfig(context *clusterd.DaemonContext, config map[string]int) error {
+	if len(config) == 0 {
+		return nil
+	}
+
+	b, err := json.Marshal(config)
+	if err != nil {
+		return err
+	}
+	filePath := path.Join(context.ConfigDir, dirOSDConfigFilename)
+	err = ioutil.WriteFile(filePath, b, 0644)
+	if err != nil {
+		return err
+	}
+
+	logger.Debugf("saved osd dir config to %s", filePath)
+	return nil
+}

pkg/cephmgr/osd/daemon_test.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2016 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package osd
+
+import (
+	"testing"
+
+	"os"
+
+	"github.com/coreos/pkg/capnslog"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStoreOSDDirMap(t *testing.T) {
+	context := clusterd.NewDaemonContext("/tmp/testdir", "", capnslog.INFO)
+	defer os.RemoveAll(context.ConfigDir)
+	os.MkdirAll(context.ConfigDir, 0755)
+
+	dirMap, err := getDataDirs(context, false)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(dirMap))
+
+	dirMap, err = getDataDirs(context, true)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(dirMap))
+	assert.Equal(t, unassignedOSDID, dirMap[context.ConfigDir])
+	dirMap[context.ConfigDir] = 0
+
+	err = saveDirConfig(context, dirMap)
+	assert.Nil(t, err)
+
+	dirMap, err = getDataDirs(context, false)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(dirMap))
+	assert.Equal(t, 0, dirMap[context.ConfigDir])
+
+	// add another directory to the map
+	dirMap["/tmp/mydir"] = 23
+	err = saveDirConfig(context, dirMap)
+	assert.Nil(t, err)
+
+	dirMap, err = getDataDirs(context, false)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(dirMap))
+	assert.Equal(t, 0, dirMap[context.ConfigDir])
+	assert.Equal(t, 23, dirMap["/tmp/mydir"])
+}

pkg/operator/operator.go

@@ -36,14 +36,16 @@ type Operator struct {
 	clientset        *kubernetes.Clientset
 	waitCluster      sync.WaitGroup
 	factory          client.ConnectionFactory
+	useAllDevices    bool
 }
 
-func New(namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset, containerVersion string) *Operator {
+func New(namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset, containerVersion string, useAllDevices bool) *Operator {
 	return &Operator{
 		Namespace:        namespace,
 		factory:          factory,
 		clientset:        clientset,
 		containerVersion: containerVersion,
+		useAllDevices:    useAllDevices,
 	}
 }
 
@@ -63,7 +65,7 @@ func (o *Operator) Run() error {
 	}
 
 	// Start the OSDs
-	osds := osd.New(o.Namespace, o.containerVersion)
+	osds := osd.New(o.Namespace, o.containerVersion, o.useAllDevices)
 	err = osds.Start(o.clientset, cluster)
 	if err != nil {
 		return fmt.Errorf("failed to start the osds. %+v", err)

pkg/operator/operator_test.go

@@ -18,5 +18,5 @@ package operator
 import "testing"
 
 func TestOperator(t *testing.T) {
-	New("foo", nil, nil, "version")
+	New("foo", nil, nil, "version", true)
 }

pkg/operator/osd/osd.go

@@ -31,15 +31,17 @@ const (
 )
 
 type Cluster struct {
-	Namespace string
-	Keyring   string
-	Version   string
+	Namespace     string
+	Keyring       string
+	Version       string
+	useAllDevices bool
 }
 
-func New(namespace, version string) *Cluster {
+func New(namespace, version string, useAllDevices bool) *Cluster {
 	return &Cluster{
-		Namespace: namespace,
-		Version:   version,
+		Namespace:     namespace,
+		Version:       version,
+		useAllDevices: useAllDevices,
 	}
 }
 
@@ -97,6 +99,10 @@ func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) v1.Container {
 
 	command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s ",
 		k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name)
+	if c.useAllDevices {
+		command += fmt.Sprintf("--data-devices=all ")
+	}
+
 	privileged := true
 	return v1.Container{
 		// TODO: fix "sleep 5".