From a07afb2e10f3d9d0f478548d01ecb16674df7acf Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Wed, 15 Feb 2017 14:13:12 -0800 Subject: [PATCH] flag to allow osds to use all devices --- cmd/rook-operator/main.go | 5 ++++- cmd/rookd/osd.go | 9 ++++++--- pkg/cephmgr/osd/daemon.go | 11 +++++++---- pkg/operator/operator.go | 6 ++++-- pkg/operator/operator_test.go | 2 +- pkg/operator/osd/osd.go | 18 ++++++++++++------ 6 files changed, 34 insertions(+), 17 deletions(-) diff --git a/cmd/rook-operator/main.go b/cmd/rook-operator/main.go index 9a91e2831..dd94b264b 100644 --- a/cmd/rook-operator/main.go +++ b/cmd/rook-operator/main.go @@ -47,6 +47,7 @@ type config struct { cephConfigOverride string clusterInfo mon.ClusterInfo monEndpoints string + useAllDevices bool } var logLevelRaw string @@ -74,6 +75,8 @@ func init() { rootCmd.PersistentFlags().StringVar(&cfg.dataDir, "data-dir", "/var/lib/rook", "directory for storing configuration") rootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "INFO", "logging level for logging/tracing output (valid values: CRITICAL,ERROR,WARNING,NOTICE,INFO,DEBUG,TRACE)") + rootCmd.Flags().BoolVar(&cfg.useAllDevices, "use-all-devices", false, "true to use all storage devices, false to require local device selection") + // load the environment variables flags.SetFlagsFromEnv(rootCmd.Flags(), "ROOK_OPERATOR") flags.SetFlagsFromEnv(rootCmd.PersistentFlags(), "ROOK_OPERATOR") @@ -96,7 +99,7 @@ func startOperator(cmd *cobra.Command, args []string) error { } logger.Infof("starting operator. containerVersion=%s", cfg.containerVersion) - op := operator.New(k8sutil.Namespace, cephd.New(), clientset, cfg.containerVersion) + op := operator.New(k8sutil.Namespace, cephd.New(), clientset, cfg.containerVersion, cfg.useAllDevices) err = op.Run() if err != nil { fmt.Printf("failed to run operator. %+v\n", err) diff --git a/cmd/rookd/osd.go b/cmd/rookd/osd.go index 709d95fe0..af2b10cb6 100644 --- a/cmd/rookd/osd.go +++ b/cmd/rookd/osd.go @@ -33,10 +33,14 @@ var osdCmd = &cobra.Command{ Hidden: true, } var ( - osdCluster mon.ClusterInfo + osdCluster mon.ClusterInfo + osdDataDevices string ) func init() { + osdCmd.Flags().StringVar(&osdDataDevices, "data-devices", "", "the device names to use, or \"all\"") + flags.SetFlagsFromEnv(monCmd.Flags(), "ROOKD") + osdCmd.RunE = startOSD } @@ -47,13 +51,12 @@ func startOSD(cmd *cobra.Command, args []string) error { setLogLevel() - devices := "" metadataDevice := "" forceFormat := false location := "" bluestoreConfig := partition.BluestoreConfig{DatabaseSizeMB: 512} // FIX clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) - agent := osd.NewAgent(cephd.New(), devices, metadataDevice, forceFormat, location, bluestoreConfig, &clusterInfo) + agent := osd.NewAgent(cephd.New(), osdDataDevices, metadataDevice, forceFormat, location, bluestoreConfig, &clusterInfo) context := clusterd.NewDaemonContext(cfg.dataDir, cfg.cephConfigOverride, cfg.logLevel) return osd.Run(context, agent) diff --git a/pkg/cephmgr/osd/daemon.go b/pkg/cephmgr/osd/daemon.go index 7d9468ddd..525856082 100644 --- a/pkg/cephmgr/osd/daemon.go +++ b/pkg/cephmgr/osd/daemon.go @@ -61,7 +61,7 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error { } // initialize the desired osds - devices, err := getAvailableDevices(dcontext, hardware.Disks) + devices, err := getAvailableDevices(dcontext, hardware.Disks, agent.devices) if err != nil { return fmt.Errorf("failed to get available devices. %+v", err) } @@ -93,7 +93,7 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error { return nil } -func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk) (*DeviceOsdMapping, error) { +func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk, desiredDevices string) (*DeviceOsdMapping, error) { available := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}} for _, device := range devices { if device.Type == sys.PartType { @@ -104,8 +104,11 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err) } if fs == "" && ownPartitions { - logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name) - //available.Entries[device.Name] = &DeviceOsdIDEntry{Data: (for existing osds get their ids) unassignedOSDID} + if desiredDevices == "all" { + available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} + } else { + logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name) + } } } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 8d1b47c19..1fd67cad2 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -36,14 +36,16 @@ type Operator struct { clientset *kubernetes.Clientset waitCluster sync.WaitGroup factory client.ConnectionFactory + useAllDevices bool } -func New(namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset, containerVersion string) *Operator { +func New(namespace string, factory client.ConnectionFactory, clientset *kubernetes.Clientset, containerVersion string, useAllDevices bool) *Operator { return &Operator{ Namespace: namespace, factory: factory, clientset: clientset, containerVersion: containerVersion, + useAllDevices: useAllDevices, } } @@ -63,7 +65,7 @@ func (o *Operator) Run() error { } // Start the OSDs - osds := osd.New(o.Namespace, o.containerVersion) + osds := osd.New(o.Namespace, o.containerVersion, o.useAllDevices) err = osds.Start(o.clientset, cluster) if err != nil { return fmt.Errorf("failed to start the osds. %+v", err) diff --git a/pkg/operator/operator_test.go b/pkg/operator/operator_test.go index add518320..4773f9685 100644 --- a/pkg/operator/operator_test.go +++ b/pkg/operator/operator_test.go @@ -18,5 +18,5 @@ package operator import "testing" func TestOperator(t *testing.T) { - New("foo", nil, nil, "version") + New("foo", nil, nil, "version", true) } diff --git a/pkg/operator/osd/osd.go b/pkg/operator/osd/osd.go index 7cd5ce1db..e262ec988 100644 --- a/pkg/operator/osd/osd.go +++ b/pkg/operator/osd/osd.go @@ -31,15 +31,17 @@ const ( ) type Cluster struct { - Namespace string - Keyring string - Version string + Namespace string + Keyring string + Version string + useAllDevices bool } -func New(namespace, version string) *Cluster { +func New(namespace, version string, useAllDevices bool) *Cluster { return &Cluster{ - Namespace: namespace, - Version: version, + Namespace: namespace, + Version: version, + useAllDevices: useAllDevices, } } @@ -97,6 +99,10 @@ func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) v1.Container { command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s ", k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name) + if c.useAllDevices { + command += fmt.Sprintf("--data-devices=all ") + } + privileged := true return v1.Container{ // TODO: fix "sleep 5".