diff --git a/Documentation/standalone.md b/Documentation/standalone.md index fdda258f5..af984e9d6 100644 --- a/Documentation/standalone.md +++ b/Documentation/standalone.md @@ -17,7 +17,7 @@ Rook can be deployed as a standalone service directly on any modern Linux host b 2. Start a one node Rook cluster ```bash - ./rookd --data-dir /tmp/rook-test + ./rookd --config-dir /tmp/rook-test ``` 3. In another console (but in the same path), verify the Rook cluster is ready diff --git a/build/release/containers.sh b/build/release/containers.sh index f6c8c4a73..5c58a2618 100755 --- a/build/release/containers.sh +++ b/build/release/containers.sh @@ -15,7 +15,7 @@ build() { trap "rm -fr $tmpdir" EXIT cat < $tmpdir/Dockerfile FROM alpine:3.5 -RUN apk add --no-cache gptfdisk util-linux coreutils +RUN apk add --no-cache gptfdisk util-linux coreutils e2fsprogs COPY root / ENTRYPOINT ["/usr/bin/rookd"] EOF diff --git a/cmd/rook/filesystem/mount_test.go b/cmd/rook/filesystem/mount_test.go index 80ca997fe..95c5b40d6 100644 --- a/cmd/rook/filesystem/mount_test.go +++ b/cmd/rook/filesystem/mount_test.go @@ -38,7 +38,8 @@ func TestMountFilesystem(t *testing.T) { } e := &exectest.MockExecutor{ MockExecuteCommand: func(actionName string, command string, arg ...string) error { - expectedArgs := []string{"mount", "-t", "ceph", "-o", "name=admin,secret=AQBsCv1X5oD9GhAARHVU9N+kFRWDjyLA1dqzIg==", + assert.Equal(t, "mount", command) + expectedArgs := []string{"-t", "ceph", "-o", "name=admin,secret=AQBsCv1X5oD9GhAARHVU9N+kFRWDjyLA1dqzIg==", "10.37.129.214:6790:/", "/tmp/myfs1mount"} assert.Equal(t, expectedArgs, arg) return nil diff --git a/cmd/rook/filesystem/unmount_test.go b/cmd/rook/filesystem/unmount_test.go index 0aeee1113..a71691631 100644 --- a/cmd/rook/filesystem/unmount_test.go +++ b/cmd/rook/filesystem/unmount_test.go @@ -27,7 +27,8 @@ import ( func TestUnmountFilesystem(t *testing.T) { e := &exectest.MockExecutor{ MockExecuteCommand: func(actionName string, command string, arg ...string) error { - expectedArgs := []string{"umount", "/tmp/myfs1mount"} + assert.Equal(t, "umount", command) + expectedArgs := []string{"/tmp/myfs1mount"} assert.Equal(t, expectedArgs, arg) return nil }, diff --git a/cmd/rookd/api.go b/cmd/rookd/api.go index fdf13d008..ca8b2e63d 100644 --- a/cmd/rookd/api.go +++ b/cmd/rookd/api.go @@ -50,7 +50,7 @@ func init() { } func startAPI(cmd *cobra.Command, args []string) error { - if err := flags.VerifyRequiredFlags(apiCmd, []string{"repo-prefix", "namespace", "data-dir", "cluster-name", "mon-endpoints", "mon-secret", "admin-secret"}); err != nil { + if err := flags.VerifyRequiredFlags(apiCmd, []string{"repo-prefix", "namespace", "config-dir", "cluster-name", "mon-endpoints", "mon-secret", "admin-secret"}); err != nil { return err } if apiPort == 0 { diff --git a/cmd/rookd/main.go b/cmd/rookd/main.go index 2fe03a679..3554bd9b8 100644 --- a/cmd/rookd/main.go +++ b/cmd/rookd/main.go @@ -33,7 +33,7 @@ import ( "github.com/rook/rook/pkg/cephmgr" "github.com/rook/rook/pkg/cephmgr/cephd" "github.com/rook/rook/pkg/cephmgr/mon" - "github.com/rook/rook/pkg/cephmgr/osd/partition" + "github.com/rook/rook/pkg/cephmgr/osd" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/model" "github.com/rook/rook/pkg/util" @@ -62,6 +62,7 @@ type config struct { discoveryURL string etcdMembers string devices string + directories string metadataDevice string dataDir string adminSecret string @@ -69,7 +70,7 @@ type config struct { location string logLevel capnslog.LogLevel cephConfigOverride string - bluestoreConfig partition.BluestoreConfig + storeConfig osd.StoreConfig networkInfo clusterd.NetworkInfo monEndpoints string namespace string @@ -160,7 +161,8 @@ func joinCluster() error { } services := []*clusterd.ClusterService{ - cephmgr.NewCephService(cephd.New(), cfg.devices, cfg.metadataDevice, cfg.forceFormat, cfg.location, cfg.adminSecret, cfg.bluestoreConfig), + cephmgr.NewCephService(cephd.New(), cfg.devices, cfg.metadataDevice, cfg.directories, + cfg.forceFormat, cfg.location, cfg.adminSecret, cfg.storeConfig), } cfg.nodeID, err = util.LoadPersistedNodeID(cfg.dataDir) diff --git a/cmd/rookd/mon.go b/cmd/rookd/mon.go index 46818af4a..2feccd1af 100644 --- a/cmd/rookd/mon.go +++ b/cmd/rookd/mon.go @@ -46,7 +46,7 @@ func addCephFlags(command *cobra.Command) { command.Flags().StringVar(&clusterInfo.MonitorSecret, "mon-secret", "", "the cephx keyring for monitors") command.Flags().StringVar(&clusterInfo.AdminSecret, "admin-secret", "", "secret for the admin user (random if not specified)") command.Flags().StringVar(&cfg.monEndpoints, "mon-endpoints", "", "ceph mon endpoints") - command.Flags().StringVar(&cfg.dataDir, "data-dir", "/var/lib/rook", "directory for storing configuration") + command.Flags().StringVar(&cfg.dataDir, "config-dir", "/var/lib/rook", "directory for storing configuration") command.Flags().StringVar(&cfg.cephConfigOverride, "ceph-config-override", "", "optional path to a ceph config file that will be appended to the config files that rook generates") } @@ -61,7 +61,7 @@ func init() { } func startMon(cmd *cobra.Command, args []string) error { - if err := flags.VerifyRequiredFlags(monCmd, []string{"name", "fsid", "mon-secret", "admin-secret", "data-dir", "cluster-name", "private-ipv4"}); err != nil { + if err := flags.VerifyRequiredFlags(monCmd, []string{"name", "fsid", "mon-secret", "admin-secret", "config-dir", "cluster-name", "private-ipv4"}); err != nil { return err } diff --git a/cmd/rookd/osd.go b/cmd/rookd/osd.go index 34de82fba..60246f02f 100644 --- a/cmd/rookd/osd.go +++ b/cmd/rookd/osd.go @@ -18,10 +18,11 @@ limitations under the License. package main import ( + "fmt" + "github.com/rook/rook/pkg/cephmgr/cephd" "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/cephmgr/osd" - "github.com/rook/rook/pkg/cephmgr/osd/partition" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" @@ -33,18 +34,24 @@ var osdCmd = &cobra.Command{ Hidden: true, } var ( - osdCluster mon.ClusterInfo + osdCluster mon.ClusterInfo + osdDataDeviceFilter string ) func addOSDFlags(command *cobra.Command) { - command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement") - command.Flags().StringVar(&cfg.devices, "data-devices", "", "comma separated list of devices to use for storage, or \"all\"") + command.Flags().StringVar(&cfg.devices, "data-devices", "", "comma separated list of devices to use for storage") + command.Flags().StringVar(&osdDataDeviceFilter, "data-device-filter", "", "a regex filter for the device names to use, or \"all\"") + command.Flags().StringVar(&cfg.directories, "data-directories", "", "comma separated list of directory paths to use for storage") command.Flags().StringVar(&cfg.metadataDevice, "metadata-device", "", "device to use for metadata (e.g. a high performance SSD/NVMe device)") + command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement") command.Flags().BoolVar(&cfg.forceFormat, "force-format", false, "true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!") - // bluestore config flags - command.Flags().IntVar(&cfg.bluestoreConfig.WalSizeMB, "osd-wal-size", partition.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL)") - command.Flags().IntVar(&cfg.bluestoreConfig.DatabaseSizeMB, "osd-database-size", partition.DBDefaultSizeMB, "default size (MB) for OSD database") + + // OSD store config flags + command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osd.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)") + command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", osd.DBDefaultSizeMB, "default size (MB) for OSD database (bluestore)") + command.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osd.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)") + command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", osd.Filestore, "type of backing OSD store to use (bluestore or filestore)") } func init() { @@ -60,13 +67,25 @@ func startOSD(cmd *cobra.Command, args []string) error { return err } + var dataDevices string + var usingDeviceFilter bool + if osdDataDeviceFilter != "" { + if cfg.devices != "" { + return fmt.Errorf("Only one of --data-devices and --data-device-filter can be specified.") + } + + dataDevices = osdDataDeviceFilter + usingDeviceFilter = true + } else { + dataDevices = cfg.devices + } + setLogLevel() - metadataDevice := "" forceFormat := false - location := "" clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) - agent := osd.NewAgent(cephd.New(), cfg.devices, metadataDevice, forceFormat, location, cfg.bluestoreConfig, &clusterInfo) + agent := osd.NewAgent(cephd.New(), dataDevices, usingDeviceFilter, cfg.metadataDevice, cfg.directories, forceFormat, + cfg.location, cfg.storeConfig, &clusterInfo) context := clusterd.NewDaemonContext(cfg.dataDir, cfg.cephConfigOverride, cfg.logLevel) return osd.Run(context, agent) diff --git a/glide.lock b/glide.lock index d974f2727..ba86b3313 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 81178d6bc6321564bed3985f0746ccd7357067d5c9eaecfdccdbd86cfd897387 -updated: 2017-03-08T10:56:46.619273095-08:00 +hash: 23db89af71e3e11cfa7bb7722f8dd338f3285dd87270f8e6000898662e7def08 +updated: 2017-04-03T13:42:53.811824499-07:00 imports: - name: bitbucket.org/ww/goautoneg version: a547fc61f48d567d5b4ec6f8aee5573d8efce11d @@ -150,7 +150,7 @@ imports: - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 - name: github.com/jonboulle/clockwork - version: 72f9bd7c4e0c2a40055ab3d0f09654f730cce982 + version: 2eee05ed794112d45db504eb05aa693efd2b8b09 - name: github.com/juju/ratelimit version: 77ed1c8a01217656d2080ad51981f6e99adaa177 - name: github.com/mailru/easyjson @@ -198,13 +198,12 @@ imports: - name: github.com/xiang90/probing version: 07dd2e8dfe18522e9c447ba95f2fe95262f63bb2 - name: golang.org/x/crypto - version: 1f22c0103821b9390939b6776727195525381532 + version: 1351f936d976c60a0a48d728281922cf63eafb8d subpackages: - bcrypt - blowfish - - ssh/terminal - name: golang.org/x/net - version: e90d6d0afc4c315a0d87a568ae68577cc15149a0 + version: 6acef71eb69611914f7a30939ea9f6e194c78172 subpackages: - context - context/ctxhttp @@ -212,7 +211,6 @@ imports: - http2/hpack - idna - internal/timeseries - - lex/httplex - trace - name: golang.org/x/oauth2 version: 3c3a985cb79f52a3190fbc056984415ca6763d01 @@ -222,7 +220,7 @@ imports: - jws - jwt - name: golang.org/x/sys - version: 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9 + version: 9c60d1c508f5134d1ca726b4641db998f2523357 subpackages: - unix - name: golang.org/x/text @@ -269,19 +267,33 @@ imports: version: e121606b0d09b2e1c467183ee46217fa85a6b672 subpackages: - discovery + - discovery/fake - kubernetes + - kubernetes/fake - kubernetes/typed/apps/v1beta1 + - kubernetes/typed/apps/v1beta1/fake - kubernetes/typed/authentication/v1beta1 + - kubernetes/typed/authentication/v1beta1/fake - kubernetes/typed/authorization/v1beta1 + - kubernetes/typed/authorization/v1beta1/fake - kubernetes/typed/autoscaling/v1 + - kubernetes/typed/autoscaling/v1/fake - kubernetes/typed/batch/v1 + - kubernetes/typed/batch/v1/fake - kubernetes/typed/batch/v2alpha1 + - kubernetes/typed/batch/v2alpha1/fake - kubernetes/typed/certificates/v1alpha1 + - kubernetes/typed/certificates/v1alpha1/fake - kubernetes/typed/core/v1 + - kubernetes/typed/core/v1/fake - kubernetes/typed/extensions/v1beta1 + - kubernetes/typed/extensions/v1beta1/fake - kubernetes/typed/policy/v1beta1 + - kubernetes/typed/policy/v1beta1/fake - kubernetes/typed/rbac/v1alpha1 + - kubernetes/typed/rbac/v1alpha1/fake - kubernetes/typed/storage/v1beta1 + - kubernetes/typed/storage/v1beta1/fake - pkg/api - pkg/api/errors - pkg/api/install @@ -370,6 +382,7 @@ imports: - plugin/pkg/client/auth/gcp - plugin/pkg/client/auth/oidc - rest + - testing - tools/clientcmd/api - tools/metrics - transport diff --git a/glide.yaml b/glide.yaml index 183bd031b..df8de33ac 100644 --- a/glide.yaml +++ b/glide.yaml @@ -19,6 +19,7 @@ import: - package: github.com/spf13/pflag - package: github.com/google/uuid - package: github.com/go-ini/ini +- package: github.com/ghodss/yaml - package: github.com/gorilla/mux - package: bitbucket.org/ww/goautoneg version: a547fc61f48d567d5b4ec6f8aee5573d8efce11d diff --git a/pkg/cephmgr/cephmgr.go b/pkg/cephmgr/cephmgr.go index ec6627e94..67a3a5a93 100644 --- a/pkg/cephmgr/cephmgr.go +++ b/pkg/cephmgr/cephmgr.go @@ -20,7 +20,6 @@ import ( "github.com/rook/rook/pkg/cephmgr/mds" "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/cephmgr/osd" - "github.com/rook/rook/pkg/cephmgr/osd/partition" "github.com/rook/rook/pkg/cephmgr/rgw" "github.com/rook/rook/pkg/clusterd" ) @@ -30,15 +29,15 @@ const ( ) // create a new ceph service -func NewCephService(factory client.ConnectionFactory, devices, metadataDevice string, forceFormat bool, - location, adminSecret string, bluestoreConfig partition.BluestoreConfig) *clusterd.ClusterService { +func NewCephService(factory client.ConnectionFactory, devices, metadataDevice, directories string, forceFormat bool, + location, adminSecret string, storeConfig osd.StoreConfig) *clusterd.ClusterService { return &clusterd.ClusterService{ Name: cephName, Leader: newLeader(factory, adminSecret), Agents: []clusterd.ServiceAgent{ mon.NewAgent(factory), - osd.NewAgent(factory, devices, metadataDevice, forceFormat, location, bluestoreConfig, nil), + osd.NewAgent(factory, devices, false, metadataDevice, directories, forceFormat, location, storeConfig, nil), mds.NewAgent(factory), rgw.NewAgent(factory), }, diff --git a/pkg/cephmgr/leader_test.go b/pkg/cephmgr/leader_test.go index 87738727b..6b5368d60 100644 --- a/pkg/cephmgr/leader_test.go +++ b/pkg/cephmgr/leader_test.go @@ -22,7 +22,7 @@ import ( testceph "github.com/rook/rook/pkg/cephmgr/client/test" "github.com/rook/rook/pkg/cephmgr/mon" - "github.com/rook/rook/pkg/cephmgr/osd/partition" + "github.com/rook/rook/pkg/cephmgr/osd" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/clusterd/inventory" "github.com/rook/rook/pkg/util" @@ -138,7 +138,7 @@ func TestRefreshKeys(t *testing.T) { func TestNewCephService(t *testing.T) { factory := &testceph.MockConnectionFactory{Fsid: "fsid", SecretKey: "key"} - service := NewCephService(factory, "a,b,c", "", true, "root=default", "", partition.BluestoreConfig{}) + service := NewCephService(factory, "a,b,c", "", "", true, "root=default", "", osd.StoreConfig{}) assert.NotNil(t, service) assert.Equal(t, "/rook/services/ceph/osd/desired", service.Leader.RefreshKeys()[0].Path) assert.Equal(t, 4, len(service.Agents)) diff --git a/pkg/cephmgr/osd/agent.go b/pkg/cephmgr/osd/agent.go index 2d30e554e..9996d8429 100644 --- a/pkg/cephmgr/osd/agent.go +++ b/pkg/cephmgr/osd/agent.go @@ -35,7 +35,6 @@ import ( "github.com/rook/rook/pkg/cephmgr/client" "github.com/rook/rook/pkg/cephmgr/mon" - "github.com/rook/rook/pkg/cephmgr/osd/partition" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/util" "github.com/rook/rook/pkg/util/proc" @@ -53,24 +52,27 @@ const ( ) type OsdAgent struct { - cluster *mon.ClusterInfo - forceFormat bool - location string - factory client.ConnectionFactory - osdProc map[int]*proc.MonitoredProc - desiredDevices []string - devices string - metadataDevice string - bluestoreConfig partition.BluestoreConfig - configCounter int32 - osdsCompleted chan struct{} + cluster *mon.ClusterInfo + forceFormat bool + location string + factory client.ConnectionFactory + osdProc map[int]*proc.MonitoredProc + desiredDevices []string + desiredDirectories []string + devices string + usingDeviceFilter bool + metadataDevice string + directories string + storeConfig StoreConfig + configCounter int32 + osdsCompleted chan struct{} } -func NewAgent(factory client.ConnectionFactory, devices, metadataDevice string, forceFormat bool, - location string, bluestoreConfig partition.BluestoreConfig, cluster *mon.ClusterInfo) *OsdAgent { +func NewAgent(factory client.ConnectionFactory, devices string, usingDeviceFilter bool, metadataDevice, directories string, forceFormat bool, + location string, storeConfig StoreConfig, cluster *mon.ClusterInfo) *OsdAgent { - return &OsdAgent{factory: factory, devices: devices, metadataDevice: metadataDevice, forceFormat: forceFormat, - location: location, bluestoreConfig: bluestoreConfig, cluster: cluster} + return &OsdAgent{factory: factory, devices: devices, usingDeviceFilter: usingDeviceFilter, metadataDevice: metadataDevice, + directories: directories, forceFormat: forceFormat, location: location, storeConfig: storeConfig, cluster: cluster} } func (a *OsdAgent) Name() string { @@ -86,13 +88,24 @@ func (a *OsdAgent) Initialize(context *clusterd.Context) error { logger.Infof("desired devices for osds: %+v. desired metadata device: %s", a.desiredDevices, a.metadataDevice) } + if len(a.directories) > 0 { + a.desiredDirectories = strings.Split(a.directories, ",") + } + // if no devices or directories were specified, use the current directory for an osd - if len(a.desiredDevices) == 0 { + if len(a.desiredDevices) == 0 && len(a.desiredDirectories) == 0 { logger.Infof("Adding local path %s to desired state", context.ConfigDir) - err := AddDesiredDir(context.EtcdClient, context.ConfigDir, context.NodeID) - if err != nil { - return fmt.Errorf("failed to add current dir %s. %v", context.ConfigDir, err) + a.desiredDirectories = []string{context.ConfigDir} + } + + if len(a.desiredDirectories) > 0 { + for _, dir := range a.desiredDirectories { + err := AddDesiredDir(context.EtcdClient, dir, context.NodeID) + if err != nil { + return fmt.Errorf("failed to add desired dir %s. %v", dir, err) + } } + logger.Infof("desired directories for osds: %+v.", a.desiredDirectories) } return nil @@ -351,7 +364,7 @@ func (a *OsdAgent) configureDirs(context *clusterd.Context, dirs map[string]int) } } - logger.Infof("%d/%d osds (filestore) succeeded on this node", succeeded, len(dirs)) + logger.Infof("%d/%d osd dirs succeeded on this node", succeeded, len(dirs)) return lastErr } @@ -387,7 +400,7 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs defer a.decrementConfigCounter() // compute an OSD layout scheme that will optimize performance - scheme, err := getPartitionPerfScheme(context, connection, devices, a.bluestoreConfig) + scheme, err := getPartitionPerfScheme(context, connection, devices, a.storeConfig) logger.Debugf("partition scheme: %+v, err: %+v", scheme, err) if err != nil { logger.Errorf("failed to get OSD performance scheme: %+v", err) @@ -396,8 +409,8 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs if scheme.Metadata != nil { // partition the dedicated metadata device - if err := partitionBluestoreMetadata(context, scheme.Metadata, context.ConfigDir); err != nil { - logger.Errorf("failed to partition bluestore metadata %+v: %+v", scheme.Metadata, err) + if err := partitionMetadata(context, scheme.Metadata, context.ConfigDir); err != nil { + logger.Errorf("failed to partition metadata %+v: %+v", scheme.Metadata, err) return } } @@ -414,7 +427,7 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs } } - logger.Infof("%d/%d bluestore osds succeeded on this node", succeeded, len(scheme.Entries)) + logger.Infof("%d/%d osd devices succeeded on this node", succeeded, len(scheme.Entries)) }() return nil @@ -423,10 +436,10 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs // computes a partitioning scheme for all the given desired devices. This could be devics already in use, // devices dedicated to metadata, and devices with all bluestore partitions collocated. func getPartitionPerfScheme(context *clusterd.Context, connection client.Connection, devices *DeviceOsdMapping, - bluestoreConfig partition.BluestoreConfig) (*partition.PerfScheme, error) { + storeConfig StoreConfig) (*PerfScheme, error) { // load the existing (committed) partition scheme from disk - perfScheme, err := partition.LoadScheme(context.ConfigDir) + perfScheme, err := LoadScheme(context.ConfigDir) if err != nil { return nil, fmt.Errorf("failed to load partition scheme from %s: %+v", context.ConfigDir, err) } @@ -461,7 +474,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect } metadataEntry = mapping - perfScheme.Metadata = partition.NewMetadataDeviceInfo(name) + perfScheme.Metadata = NewMetadataDeviceInfo(name) } } @@ -478,7 +491,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect return nil, fmt.Errorf("failed to register OSD for device %s: %+v", name, err) } - schemeEntry := partition.NewPerfSchemeEntry() + schemeEntry := NewPerfSchemeEntry(storeConfig.StoreType) schemeEntry.ID = *osdID schemeEntry.OsdUUID = *osdUUID @@ -488,7 +501,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect mapping.Data = *osdID // populate the perf partition scheme entry with distributed partition details - err := partition.PopulateDistributedPerfSchemeEntry(schemeEntry, name, perfScheme.Metadata, bluestoreConfig) + err := PopulateDistributedPerfSchemeEntry(schemeEntry, name, perfScheme.Metadata, storeConfig) if err != nil { return nil, fmt.Errorf("failed to create distributed perf scheme entry for %s: %+v", name, err) } @@ -500,7 +513,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect mapping.Metadata = []int{*osdID} // populate the perf partition scheme entry with collocated partition details - err := partition.PopulateCollocatedPerfSchemeEntry(schemeEntry, name, bluestoreConfig) + err := PopulateCollocatedPerfSchemeEntry(schemeEntry, name, storeConfig) if err != nil { return nil, fmt.Errorf("failed to create collocated perf scheme entry for %s: %+v", name, err) } @@ -514,7 +527,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect } // determines if the given device name is already in use with existing/committed partitions -func isDeviceInUse(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) bool { +func isDeviceInUse(name string, nameToUUID map[string]string, scheme *PerfScheme) bool { parts := findPartitionsForDevice(name, nameToUUID, scheme) return len(parts) > 0 } @@ -529,12 +542,12 @@ func isDeviceDesiredForData(mapping *DeviceOsdIDEntry) bool { (mapping.Data > unassignedOSDID && len(mapping.Metadata) == 1) } -func isDeviceDesiredForMetadata(mapping *DeviceOsdIDEntry, scheme *partition.PerfScheme) bool { +func isDeviceDesiredForMetadata(mapping *DeviceOsdIDEntry, scheme *PerfScheme) bool { return mapping.Data == unassignedOSDID && mapping.Metadata != nil && len(mapping.Metadata) == 0 } // finds all the partition details that are on the given device name -func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) []*partition.PerfSchemePartitionDetails { +func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *PerfScheme) []*PerfSchemePartitionDetails { if scheme == nil { return nil } @@ -544,7 +557,7 @@ func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme * return nil } - parts := []*partition.PerfSchemePartitionDetails{} + parts := []*PerfSchemePartitionDetails{} for _, e := range scheme.Entries { for _, p := range e.Partitions { if p.DiskUUID == diskUUID { @@ -558,7 +571,7 @@ func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme * // if a device name has changed, this function will find all partition entries with the device's static UUID and // then update the device name on them -func refreshDeviceInfo(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) { +func refreshDeviceInfo(name string, nameToUUID map[string]string, scheme *PerfScheme) { parts := findPartitionsForDevice(name, nameToUUID, scheme) if len(parts) == 0 { return @@ -595,7 +608,7 @@ func (a *OsdAgent) startOSD(context *clusterd.Context, connection client.Connect if newOSD { if config.partitionScheme != nil { // format and partition the device if needed - savedScheme, err := partition.LoadScheme(config.configRoot) + savedScheme, err := LoadScheme(config.configRoot) if err != nil { return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", config.configRoot, err) } @@ -610,39 +623,25 @@ func (a *OsdAgent) startOSD(context *clusterd.Context, connection client.Connect } if !skipFormat { - err = formatDevice(context, config, a.forceFormat) + err = formatDevice(context, config, a.forceFormat, a.storeConfig) if err != nil { return fmt.Errorf("failed format/partition of osd %d. %+v", config.id, err) } - logger.Notice("waiting after bluestore partition/format...") + logger.Notice("waiting after partition/format...") <-time.After(2 * time.Second) } } // osd_data_dir/whoami does not exist yet, create/initialize the OSD - err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, context.Executor) + err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, a.storeConfig, context.Executor) if err != nil { return fmt.Errorf("failed to initialize OSD at %s: %+v", config.rootPath, err) } // save the osd information to applied state - dataDiskUUID, metadataDiskUUID := "", "" - if config.partitionScheme != nil { - dataDiskUUID = config.partitionScheme.Partitions[partition.BlockPartitionName].DiskUUID - metadataDiskUUID = config.partitionScheme.Partitions[partition.DatabasePartitionName].DiskUUID - } - - if context.EtcdClient != nil { - settings := map[string]string{ - "path": config.configRoot, - dataDiskUUIDKey: dataDiskUUID, - metadataDiskUUIDKey: metadataDiskUUID, - } - key := path.Join(getAppliedKey(context.NodeID), fmt.Sprintf("%d", config.id)) - if err := util.StoreEtcdProperties(context.EtcdClient, key, settings); err != nil { - return fmt.Errorf("failed to mark osd %d as applied: %+v", config.id, err) - } + if err := markOSDAsApplied(context, config); err != nil { + return fmt.Errorf("failed to mark osd %d as applied: %+v", config.id, err) } } else { // osd_data_dir/whoami already exists, meaning the OSD is already set up. @@ -680,7 +679,7 @@ func (a *OsdAgent) runOSD(context *clusterd.Context, clusterName string, config osdUUIDArg, } - if config.dir { + if !isBluestore(config) { params = append(params, fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(config.rootPath))) } @@ -854,6 +853,40 @@ func associateOSDIDsWithMetadataDevice(etcdClient etcd.KeysAPI, nodeID, name, id return err } +func markOSDAsApplied(context *clusterd.Context, config *osdConfig) error { + if context.EtcdClient == nil { + return nil + } + + dataDiskUUID, metadataDiskUUID := "", "" + if config.partitionScheme != nil { + dataPartDetails, err := getDataPartitionDetails(config) + if err != nil { + return err + } + + metadataPartDetails, err := getMetadataPartitionDetails(config) + if err != nil { + return err + } + + dataDiskUUID = dataPartDetails.DiskUUID + metadataDiskUUID = metadataPartDetails.DiskUUID + } + + settings := map[string]string{ + "path": config.configRoot, + dataDiskUUIDKey: dataDiskUUID, + metadataDiskUUIDKey: metadataDiskUUID, + } + key := path.Join(getAppliedKey(context.NodeID), fmt.Sprintf("%d", config.id)) + if err := util.StoreEtcdProperties(context.EtcdClient, key, settings); err != nil { + return err + } + + return nil +} + // add a device to the desired state func AddDesiredDevice(etcdClient etcd.KeysAPI, nodeID, uuid string, osdID int) error { key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), uuid) @@ -954,3 +987,7 @@ func loadOSDInfo(config *osdConfig) error { config.uuid = osdUUID return nil } + +func isBluestore(config *osdConfig) bool { + return !config.dir && config.partitionScheme != nil && config.partitionScheme.StoreType == Bluestore +} diff --git a/pkg/cephmgr/osd/agent_test.go b/pkg/cephmgr/osd/agent_test.go index 1854e1872..4c0f75ce7 100644 --- a/pkg/cephmgr/osd/agent_test.go +++ b/pkg/cephmgr/osd/agent_test.go @@ -28,7 +28,6 @@ import ( "github.com/google/uuid" testceph "github.com/rook/rook/pkg/cephmgr/client/test" "github.com/rook/rook/pkg/cephmgr/mon" - "github.com/rook/rook/pkg/cephmgr/osd/partition" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/clusterd/inventory" "github.com/rook/rook/pkg/util" @@ -38,6 +37,11 @@ import ( ) func TestOSDAgentWithDevices(t *testing.T) { + testOSDAgentWithDevicesHelper(t, StoreConfig{StoreType: Bluestore}) + testOSDAgentWithDevicesHelper(t, StoreConfig{StoreType: Filestore}) +} + +func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig StoreConfig) { // set up a temporary config directory that will be cleaned up after test configDir, err := ioutil.TempDir("", "TestOSDAgentWithDevices") if err != nil { @@ -47,7 +51,7 @@ func TestOSDAgentWithDevices(t *testing.T) { clusterName := "mycluster" nodeID := "abc" - etcdClient, agent, _ := createTestAgent(t, nodeID, "sdx,sdy", configDir) + etcdClient, agent, _ := createTestAgent(t, nodeID, "sdx,sdy", configDir, &storeConfig) startCount := 0 executor := &exectest.MockExecutor{} @@ -74,10 +78,10 @@ func TestOSDAgentWithDevices(t *testing.T) { nameSuffix = parts[1] } switch { - case execCount == 0: // first exec is the mkfs for sdy + case execCount == 0: // first exec is the mkfs for sdx assert.Equal(t, "--mkfs", args[3]) createTestKeyring(t, configDir, args) - case execCount == 1: // all remaining execs are for partitioning sdx then mkfs sdx + case execCount == 1: // all remaining execs are for partitioning sdy then mkfs sdy assert.Equal(t, "sgdisk", command) assert.Equal(t, "--zap-all", args[0]) assert.Equal(t, "/dev/"+nameSuffix, args[1]) @@ -86,10 +90,39 @@ func TestOSDAgentWithDevices(t *testing.T) { assert.Equal(t, "--clear", args[0]) assert.Equal(t, "/dev/"+nameSuffix, args[2]) case execCount == 3: - assert.Equal(t, "/dev/"+nameSuffix, args[10]) + // the partitioning for sdy. depending on bluestore/filestore, the args will be different. + expectedDeviceArgNum := 0 + if storeConfig.StoreType == Bluestore { + expectedDeviceArgNum = 10 + } else { + expectedDeviceArgNum = 4 + } + assert.Equal(t, "sgdisk", command) + assert.Equal(t, "/dev/"+nameSuffix, args[expectedDeviceArgNum]) case execCount == 4: - assert.Equal(t, "--mkfs", args[3]) - createTestKeyring(t, configDir, args) + if storeConfig.StoreType == Bluestore { + // the osd mkfs for sdy bluestore + assert.Equal(t, "--mkfs", args[3]) + createTestKeyring(t, configDir, args) + } else { + // mkfs.ext4 for sdy filestore + assert.Equal(t, "mkfs.ext4", command) + } + case execCount == 5: + if storeConfig.StoreType == Filestore { + // the mount for sdy filestore + assert.Equal(t, "mount", command) + } else { + assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount)) + } + case execCount == 6: + if storeConfig.StoreType == Filestore { + // the osd mkfs for sdy filestore + assert.Equal(t, "--mkfs", args[3]) + createTestKeyring(t, configDir, args) + } else { + assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount)) + } default: assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount)) } @@ -136,11 +169,16 @@ func TestOSDAgentWithDevices(t *testing.T) { <-agent.osdsCompleted assert.Equal(t, 0, agent.configCounter) - assert.Equal(t, 5, execCount) // 1 mkfs for sdy, 3 partition steps for sdx, 1 mkfs for sdx assert.Equal(t, 2, outputExecCount) assert.Equal(t, 2, startCount) // 2 OSD procs should be started assert.Equal(t, 2, len(agent.osdProc), fmt.Sprintf("procs=%+v", agent.osdProc)) + if storeConfig.StoreType == Bluestore { + assert.Equal(t, 5, execCount) // 1 osd mkfs for sdx, 3 partition steps for sdy, 1 osd mkfs for sdy + } else { + assert.Equal(t, 7, execCount) // 1 osd mkfs for sdx, 3 partition steps for sdy, 1 mkfs for sdy, 1 mount for sdy, 1 osd mkfs for sdy + } + err = agent.DestroyLocalService(context) assert.Nil(t, err) assert.Equal(t, 0, len(agent.osdProc)) @@ -159,7 +197,7 @@ func TestOSDAgentNoDevices(t *testing.T) { // create a test OSD agent with no devices specified nodeID := "abc" - etcdClient, agent, _ := createTestAgent(t, nodeID, "", configDir) + etcdClient, agent, _ := createTestAgent(t, nodeID, "", configDir, nil) startCount := 0 executor := &exectest.MockExecutor{} @@ -251,7 +289,7 @@ func TestRemoveDevice(t *testing.T) { defer os.RemoveAll(configDir) nodeID := "a" - etcdClient, agent, conn := createTestAgent(t, nodeID, "", configDir) + etcdClient, agent, conn := createTestAgent(t, nodeID, "", configDir, nil) executor := &exectest.MockExecutor{} context := &clusterd.Context{EtcdClient: etcdClient, NodeID: nodeID, Executor: executor, ProcMan: proc.New(executor), Inventory: createInventory()} @@ -272,12 +310,15 @@ func TestRemoveDevice(t *testing.T) { assert.True(t, applied.Equals(util.CreateSet([]string{"23"})), fmt.Sprintf("applied=%+v", applied)) } -func createTestAgent(t *testing.T, nodeID, devices, configDir string) (*util.MockEtcdClient, *OsdAgent, *testceph.MockConnection) { +func createTestAgent(t *testing.T, nodeID, devices, configDir string, storeConfig *StoreConfig) (*util.MockEtcdClient, *OsdAgent, *testceph.MockConnection) { location := "root=here" forceFormat := false + if storeConfig == nil { + storeConfig = &StoreConfig{StoreType: Bluestore} + } etcdClient := util.NewMockEtcdClient() factory := &testceph.MockConnectionFactory{} - agent := NewAgent(factory, devices, "", forceFormat, location, partition.BluestoreConfig{}, nil) + agent := NewAgent(factory, devices, false, "", "", forceFormat, location, *storeConfig, nil) agent.cluster = &mon.ClusterInfo{Name: "myclust"} agent.Initialize(&clusterd.Context{EtcdClient: etcdClient, NodeID: nodeID, ConfigDir: configDir}) if devices == "" { @@ -443,7 +484,7 @@ func TestGetPartitionPerfScheme(t *testing.T) { return nil, "", fmt.Errorf("unexpected mon_command '%s'", string(args)) } - scheme, err := getPartitionPerfScheme(context, conn, devices, partition.BluestoreConfig{}) + scheme, err := getPartitionPerfScheme(context, conn, devices, StoreConfig{}) assert.Nil(t, err) assert.Equal(t, 2, len(scheme.Entries)) @@ -457,11 +498,11 @@ func TestGetPartitionPerfScheme(t *testing.T) { // and that the second is the other one. entry := scheme.Entries[0] assert.Equal(t, 11, entry.ID) - firstBlockDevice := entry.Partitions[partition.BlockPartitionName].Device + firstBlockDevice := entry.Partitions[BlockPartitionType].Device assert.True(t, firstBlockDevice == "sda" || firstBlockDevice == "sdb", firstBlockDevice) - verifyPartitionEntry(t, entry.Partitions[partition.BlockPartitionName], firstBlockDevice, -1, 1) - verifyPartitionEntry(t, entry.Partitions[partition.WalPartitionName], "sdc", partition.WalDefaultSizeMB, 1) - verifyPartitionEntry(t, entry.Partitions[partition.DatabasePartitionName], "sdc", partition.DBDefaultSizeMB, 577) + verifyPartitionEntry(t, entry.Partitions[BlockPartitionType], firstBlockDevice, -1, 1) + verifyPartitionEntry(t, entry.Partitions[WalPartitionType], "sdc", WalDefaultSizeMB, 1) + verifyPartitionEntry(t, entry.Partitions[DatabasePartitionType], "sdc", DBDefaultSizeMB, 577) // verify the second entry in the scheme. Note the comment above about sda vs. sdb ordering. entry = scheme.Entries[1] @@ -472,9 +513,9 @@ func TestGetPartitionPerfScheme(t *testing.T) { } else { secondBlockDevice = "sda" } - verifyPartitionEntry(t, entry.Partitions[partition.BlockPartitionName], secondBlockDevice, -1, 1) - verifyPartitionEntry(t, entry.Partitions[partition.WalPartitionName], "sdc", partition.WalDefaultSizeMB, 21057) - verifyPartitionEntry(t, entry.Partitions[partition.DatabasePartitionName], "sdc", partition.DBDefaultSizeMB, 21633) + verifyPartitionEntry(t, entry.Partitions[BlockPartitionType], secondBlockDevice, -1, 1) + verifyPartitionEntry(t, entry.Partitions[WalPartitionType], "sdc", WalDefaultSizeMB, 21057) + verifyPartitionEntry(t, entry.Partitions[DatabasePartitionType], "sdc", DBDefaultSizeMB, 21633) } func TestGetPartitionPerfSchemeDiskInUse(t *testing.T) { @@ -508,7 +549,7 @@ func TestGetPartitionPerfSchemeDiskInUse(t *testing.T) { // get the partition scheme based on the desired devices. Since sda is already in use, the partition // scheme returned should reflect that. - scheme, err := getPartitionPerfScheme(context, nil, devices, partition.BluestoreConfig{}) + scheme, err := getPartitionPerfScheme(context, nil, devices, StoreConfig{}) assert.Nil(t, err) // the partition scheme should have a single entry for osd 1 on sda and it should have collocated data and metadata @@ -553,20 +594,20 @@ func TestGetPartitionPerfSchemeDiskNameChanged(t *testing.T) { // get the current partition scheme. This should notice that the device names changed and update the // partition scheme to have the latest device names - scheme, err := getPartitionPerfScheme(context, nil, devices, partition.BluestoreConfig{}) + scheme, err := getPartitionPerfScheme(context, nil, devices, StoreConfig{}) assert.Nil(t, err) assert.NotNil(t, scheme) assert.Equal(t, "nvme01-changed", scheme.Metadata.Device) - assert.Equal(t, "sda-changed", scheme.Entries[0].Partitions[partition.BlockPartitionName].Device) - assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[partition.WalPartitionName].Device) - assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[partition.DatabasePartitionName].Device) + assert.Equal(t, "sda-changed", scheme.Entries[0].Partitions[BlockPartitionType].Device) + assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[WalPartitionType].Device) + assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[DatabasePartitionType].Device) } func createInventory() *inventory.Config { return &inventory.Config{Local: &inventory.Hardware{Disks: []*inventory.LocalDisk{}}} } -func verifyPartitionEntry(t *testing.T, actual *partition.PerfSchemePartitionDetails, expectedDevice string, +func verifyPartitionEntry(t *testing.T, actual *PerfSchemePartitionDetails, expectedDevice string, expectedSize int64, expectedOffset int64) { assert.Equal(t, expectedDevice, actual.Device) @@ -574,12 +615,12 @@ func verifyPartitionEntry(t *testing.T, actual *partition.PerfSchemePartitionDet assert.Equal(t, expectedOffset, actual.OffsetMB) } -func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string) (entry *partition.PerfSchemeEntry, diskUUID string) { - entry = partition.NewPerfSchemeEntry() +func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string) (entry *PerfSchemeEntry, diskUUID string) { + entry = NewPerfSchemeEntry(Bluestore) entry.ID = osdID entry.OsdUUID = uuid.Must(uuid.NewRandom()) - partition.PopulateCollocatedPerfSchemeEntry(entry, device, partition.BluestoreConfig{}) - scheme := partition.NewPerfScheme() + PopulateCollocatedPerfSchemeEntry(entry, device, StoreConfig{}) + scheme := NewPerfScheme() scheme.Entries = append(scheme.Entries, entry) err := scheme.Save(configDir) assert.Nil(t, err) @@ -594,19 +635,19 @@ func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string) return entry, diskUUID } -func mockDistributedPartitionScheme(t *testing.T, osdID int, metadataDevice, device, configDir string) (*partition.PerfScheme, string, string) { - scheme := partition.NewPerfScheme() - scheme.Metadata = partition.NewMetadataDeviceInfo(metadataDevice) +func mockDistributedPartitionScheme(t *testing.T, osdID int, metadataDevice, device, configDir string) (*PerfScheme, string, string) { + scheme := NewPerfScheme() + scheme.Metadata = NewMetadataDeviceInfo(metadataDevice) - entry := partition.NewPerfSchemeEntry() + entry := NewPerfSchemeEntry(Bluestore) entry.ID = osdID entry.OsdUUID = uuid.Must(uuid.NewRandom()) - partition.PopulateDistributedPerfSchemeEntry(entry, device, scheme.Metadata, partition.BluestoreConfig{}) + PopulateDistributedPerfSchemeEntry(entry, device, scheme.Metadata, StoreConfig{}) scheme.Entries = append(scheme.Entries, entry) err := scheme.Save(configDir) assert.Nil(t, err) // return the full partition scheme, the metadata device UUID and the data device UUID - return scheme, scheme.Metadata.DiskUUID, entry.Partitions[partition.BlockPartitionName].DiskUUID + return scheme, scheme.Metadata.DiskUUID, entry.Partitions[BlockPartitionType].DiskUUID } diff --git a/pkg/cephmgr/osd/crushmap.go b/pkg/cephmgr/osd/crushmap.go index ace9890c5..1a07e0bb4 100644 --- a/pkg/cephmgr/osd/crushmap.go +++ b/pkg/cephmgr/osd/crushmap.go @@ -49,14 +49,14 @@ func formatLocation(location string) ([]string, error) { } } - if !isCrushFieldSet("hostName", pairs) { + if !isCrushFieldSet("host", pairs) { // host name isn't set yet, attempt to set a default hostName, err := os.Hostname() if err != nil { return nil, fmt.Errorf("failed to get hostname, %+v", err) } - pairs = append(pairs, formatProperty("hostName", hostName)) + pairs = append(pairs, formatProperty("host", hostName)) } // set a default root if it's not already set diff --git a/pkg/cephmgr/osd/crushmap_test.go b/pkg/cephmgr/osd/crushmap_test.go index 8a4b3ccf8..60fd66ba9 100644 --- a/pkg/cephmgr/osd/crushmap_test.go +++ b/pkg/cephmgr/osd/crushmap_test.go @@ -49,12 +49,12 @@ func TestCrushMap(t *testing.T) { argsSet := util.CreateSet(request.Args) assert.True(t, argsSet.Contains("root=default")) assert.True(t, argsSet.Contains("dc=datacenter1")) - assert.True(t, argsSet.Contains("hostName=node1")) + assert.True(t, argsSet.Contains("host=node1")) return []byte{}, "", nil } - location := "root=default,dc=datacenter1,hostName=node1" + location := "root=default,dc=datacenter1,host=node1" err := addOSDToCrushMap(conn, context, 23, "/", location) assert.Nil(t, err) @@ -84,17 +84,17 @@ func TestCrushLocation(t *testing.T) { locSet := util.CreateSet(res) assert.True(t, locSet.Contains("root=default")) assert.True(t, locSet.Contains("dc=datacenter1")) - assert.True(t, locSet.Contains(fmt.Sprintf("hostName=%s", hostName))) + assert.True(t, locSet.Contains(fmt.Sprintf("host=%s", hostName))) // test that if host name and root are already set they will be honored - loc = "root=otherRoot,dc=datacenter2,hostName=node123" + loc = "root=otherRoot,dc=datacenter2,host=node123" res, err = formatLocation(loc) assert.Nil(t, err) assert.Equal(t, 3, len(res)) locSet = util.CreateSet(res) assert.True(t, locSet.Contains("root=otherRoot")) assert.True(t, locSet.Contains("dc=datacenter2")) - assert.True(t, locSet.Contains("hostName=node123")) + assert.True(t, locSet.Contains("host=node123")) // test an invalid CRUSH location format loc = "root=default,prop:value" diff --git a/pkg/cephmgr/osd/daemon.go b/pkg/cephmgr/osd/daemon.go index a63a28017..c5ec59e04 100644 --- a/pkg/cephmgr/osd/daemon.go +++ b/pkg/cephmgr/osd/daemon.go @@ -25,6 +25,8 @@ import ( "regexp" "time" + "strings" + "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/clusterd/inventory" @@ -62,7 +64,7 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error { } // initialize the desired osds - devices, err := getAvailableDevices(dcontext, hardware.Disks, agent.devices) + devices, err := getAvailableDevices(dcontext, hardware.Disks, agent.devices, agent.usingDeviceFilter) if err != nil { return fmt.Errorf("failed to get available devices. %+v", err) } @@ -74,8 +76,8 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error { } // initialize the data directories, with the default dir if no devices were specified - useDataDir := len(devices.Entries) == 0 - dirs, err := getDataDirs(dcontext, useDataDir) + devicesSpecified := len(devices.Entries) > 0 + dirs, err := getDataDirs(dcontext, agent.directories, devicesSpecified) if err != nil { return fmt.Errorf("failed to get data dirs. %+v", err) } @@ -95,7 +97,14 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error { return nil } -func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk, desiredDevices string) (*DeviceOsdMapping, error) { +func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk, desiredDevices string, + usingDeviceFilter bool) (*DeviceOsdMapping, error) { + + var deviceList []string + if !usingDeviceFilter { + deviceList = strings.Split(desiredDevices, ",") + } + available := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}} for _, device := range devices { if device.Type == sys.PartType { @@ -109,12 +118,24 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L if desiredDevices == "all" { available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} } else if desiredDevices != "" { - // the desired devices is a regular expression - matched, err := regexp.Match(desiredDevices, []byte(device.Name)) + var matched bool + var err error + if usingDeviceFilter { + // the desired devices is a regular expression + matched, err = regexp.Match(desiredDevices, []byte(device.Name)) + } else { + for i := range deviceList { + if device.Name == deviceList[i] { + matched = true + break + } + } + } + if err == nil && matched { available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID} } else { - logger.Infof("skipping device %s that does not match the regular expression %s. %+v", device.Name, desiredDevices, err) + logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err) } } else { @@ -126,28 +147,56 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L return available, nil } -func getDataDirs(context *clusterd.DaemonContext, useDataDir bool) (map[string]int, error) { - filePath := path.Join(context.ConfigDir, dirOSDConfigFilename) - if _, err := os.Stat(filePath); os.IsNotExist(err) { - // the config file doesn't exist yet, just return the empty map unless the data dir is desired - if useDataDir { - return map[string]int{context.ConfigDir: unassignedOSDID}, nil - } - - return map[string]int{}, nil +func getDataDirs(context *clusterd.DaemonContext, desiredDirs string, devicesSpecified bool) (map[string]int, error) { + var dirList []string + if desiredDirs != "" { + dirList = strings.Split(desiredDirs, ",") } + filePath := path.Join(context.ConfigDir, dirOSDConfigFilename) + + if _, err := os.Stat(filePath); os.IsNotExist(err) { + // the config file doesn't exist yet + if len(dirList) == 0 { + if devicesSpecified { + // no dirs desired, user is using devices instead + return map[string]int{}, nil + } else { + // no devices or dirs specified, return the default data dir + return map[string]int{context.ConfigDir: unassignedOSDID}, nil + } + } + + dirMap := make(map[string]int, len(dirList)) + addDirsToDirMap(dirList, &dirMap) + return dirMap, nil + } + + // read the saved dir map from disk + var dirMap map[string]int b, err := ioutil.ReadFile(filePath) if err != nil { return nil, err } - var dirs map[string]int - err = json.Unmarshal(b, &dirs) + err = json.Unmarshal(b, &dirMap) if err != nil { return nil, err } - return dirs, nil + + // if the user has specified any directories to use, merge them into the saved config + addDirsToDirMap(dirList, &dirMap) + + return dirMap, nil +} + +func addDirsToDirMap(dirList []string, dirMap *map[string]int) { + for _, d := range dirList { + if _, ok := (*dirMap)[d]; !ok { + // the users dir isn't already in the map, add it with an unassigned ID + (*dirMap)[d] = unassignedOSDID + } + } } func saveDirConfig(context *clusterd.DaemonContext, config map[string]int) error { diff --git a/pkg/cephmgr/osd/daemon_test.go b/pkg/cephmgr/osd/daemon_test.go index 916890720..ee6a9ef5e 100644 --- a/pkg/cephmgr/osd/daemon_test.go +++ b/pkg/cephmgr/osd/daemon_test.go @@ -35,33 +35,51 @@ func TestStoreOSDDirMap(t *testing.T) { defer os.RemoveAll(context.ConfigDir) os.MkdirAll(context.ConfigDir, 0755) - dirMap, err := getDataDirs(context, false) + // user has specified devices to use, no dirs should be returned + dirMap, err := getDataDirs(context, "", true) assert.Nil(t, err) assert.Equal(t, 0, len(dirMap)) - dirMap, err = getDataDirs(context, true) + // user has no devices specified, should return default dir + dirMap, err = getDataDirs(context, "", false) assert.Nil(t, err) assert.Equal(t, 1, len(dirMap)) assert.Equal(t, unassignedOSDID, dirMap[context.ConfigDir]) - dirMap[context.ConfigDir] = 0 + // user has no devices specified but does specify dirs, those should be returned + dirMap, err = getDataDirs(context, "/rook/dir1", false) + assert.Nil(t, err) + assert.Equal(t, 1, len(dirMap)) + assert.Equal(t, unassignedOSDID, dirMap["/rook/dir1"]) + dirMap["/rook/dir1"] = 0 // simulate an OSD ID being assigned to the dir + + // save the directory config err = saveDirConfig(context, dirMap) assert.Nil(t, err) - dirMap, err = getDataDirs(context, false) + // user has specified devices to use, we should still return the saved dir + dirMap, err = getDataDirs(context, "", true) assert.Nil(t, err) assert.Equal(t, 1, len(dirMap)) - assert.Equal(t, 0, dirMap[context.ConfigDir]) + assert.Equal(t, 0, dirMap["/rook/dir1"]) - // add another directory to the map + // user has specified devices and also a directory to use. it should be added to the dir map + dirMap, err = getDataDirs(context, "/tmp/mydir", true) + assert.Nil(t, err) + assert.Equal(t, 2, len(dirMap)) + assert.Equal(t, 0, dirMap["/rook/dir1"]) + assert.Equal(t, unassignedOSDID, dirMap["/tmp/mydir"]) + + // simulate that the user's dir got an OSD by assigning it an ID dirMap["/tmp/mydir"] = 23 err = saveDirConfig(context, dirMap) assert.Nil(t, err) - dirMap, err = getDataDirs(context, false) + // user is still specifying the directory, we should get back it's ID now + dirMap, err = getDataDirs(context, "/tmp/mydir", true) assert.Nil(t, err) assert.Equal(t, 2, len(dirMap)) - assert.Equal(t, 0, dirMap[context.ConfigDir]) + assert.Equal(t, 0, dirMap["/rook/dir1"]) assert.Equal(t, 23, dirMap["/tmp/mydir"]) } @@ -99,7 +117,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="MY-PART"`, nil &inventory.LocalDisk{Name: "rdb"}, } // select all devices - mapping, err := getAvailableDevices(context, devices, "all") + mapping, err := getAvailableDevices(context, devices, "all", true) assert.Nil(t, err) assert.Equal(t, 4, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) @@ -107,26 +125,30 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="MY-PART"`, nil assert.Equal(t, -1, mapping.Entries["rda"].Data) assert.Equal(t, -1, mapping.Entries["rdb"].Data) - // select no devices - mapping, err = getAvailableDevices(context, devices, "") + // select no devices both using and not using a filter + mapping, err = getAvailableDevices(context, devices, "", false) + assert.Nil(t, err) + assert.Equal(t, 0, len(mapping.Entries)) + + mapping, err = getAvailableDevices(context, devices, "", true) assert.Nil(t, err) assert.Equal(t, 0, len(mapping.Entries)) // select the sda devices - mapping, err = getAvailableDevices(context, devices, "^sd.$") + mapping, err = getAvailableDevices(context, devices, "^sd.$", true) assert.Nil(t, err) assert.Equal(t, 2, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) // select an exact device - mapping, err = getAvailableDevices(context, devices, "sdd") + mapping, err = getAvailableDevices(context, devices, "sdd", false) assert.Nil(t, err) assert.Equal(t, 1, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sdd"].Data) // select all devices except those that have a prefix of "s" - mapping, err = getAvailableDevices(context, devices, "^[^s]") + mapping, err = getAvailableDevices(context, devices, "^[^s]", true) assert.Nil(t, err) assert.Equal(t, 2, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["rda"].Data) diff --git a/pkg/cephmgr/osd/device.go b/pkg/cephmgr/osd/device.go index ebc273393..b2ec49eb3 100644 --- a/pkg/cephmgr/osd/device.go +++ b/pkg/cephmgr/osd/device.go @@ -25,12 +25,12 @@ import ( "strconv" "strings" "syscall" + "time" "github.com/google/uuid" "github.com/rook/rook/pkg/cephmgr/client" "github.com/rook/rook/pkg/cephmgr/mon" - "github.com/rook/rook/pkg/cephmgr/osd/partition" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/clusterd/inventory" "github.com/rook/rook/pkg/util" @@ -41,6 +41,7 @@ import ( const ( DevicesValue = "devices" ForceFormatValue = "forceFormat" + diskByPartUUID = "/dev/disk/by-partuuid" cephOsdKey = mon.CephKey + "/osd" desiredOsdRootKey = cephOsdKey + "/" + clusterd.DesiredKey + "/%s" deviceDesiredKey = desiredOsdRootKey + "/device" @@ -58,7 +59,7 @@ type osdConfig struct { id int uuid uuid.UUID dir bool - partitionScheme *partition.PerfSchemeEntry + partitionScheme *PerfSchemeEntry } type Device struct { @@ -83,42 +84,42 @@ type DeviceOsdIDEntry struct { } // format the given device for usage by an OSD -func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool) error { - blockDetails, err := getBlockPartitionDetails(config) +func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool, storeConfig StoreConfig) error { + dataDetails, err := getDataPartitionDetails(config) if err != nil { return err } // check if partitions belong to rook - ownPartitions, devFS, err := checkIfDeviceAvailable(context.Executor, blockDetails.Device) + ownPartitions, devFS, err := checkIfDeviceAvailable(context.Executor, dataDetails.Device) if err != nil { return fmt.Errorf("failed to format device. %+v", err) } if !ownPartitions { if forceFormat { - logger.Warningf("device %s is being formatted, but has partitions!!", blockDetails.Device) + logger.Warningf("device %s is being formatted, but has partitions!!", dataDetails.Device) } else { - logger.Warningf("device %s has partitions that will not be formatted. Skipping device.", blockDetails.Device) + logger.Warningf("device %s has partitions that will not be formatted. Skipping device.", dataDetails.Device) } } if devFS != "" { if forceFormat { // there's a filesystem on the device, but the user has specified to force a format. give a warning about that. - logger.Warningf("device %s already formatted with %s, but forcing a format!!!", blockDetails.Device, devFS) + logger.Warningf("device %s already formatted with %s, but forcing a format!!!", dataDetails.Device, devFS) } else { // disk is already formatted and the user doesn't want to force it, but we require partitioning - return fmt.Errorf("device %s already formatted with %s", blockDetails.Device, devFS) + return fmt.Errorf("device %s already formatted with %s", dataDetails.Device, devFS) } } // format the device dangerousToFormat := !ownPartitions || devFS != "" if !dangerousToFormat || forceFormat { - err := partitionBluestoreOSD(context, config) + err := partitionOSD(context, config) if err != nil { - return fmt.Errorf("failed to partion device %s. %v", blockDetails.Device, err) + return fmt.Errorf("failed to partion device %s. %v", dataDetails.Device, err) } } @@ -157,14 +158,14 @@ func rookOwnsPartitions(partitions []*sys.Partition) bool { return true } -// partitions a given device exclusively for bluestore metadata usage -func partitionBluestoreMetadata(context *clusterd.Context, info *partition.MetadataDeviceInfo, configRoot string) error { +// partitions a given device exclusively for metadata usage +func partitionMetadata(context *clusterd.Context, info *MetadataDeviceInfo, configRoot string) error { if len(info.Partitions) == 0 { return nil } // check to see if the metadata partition scheme has already been applied - savedScheme, err := partition.LoadScheme(configRoot) + savedScheme, err := LoadScheme(configRoot) if err != nil { return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", configRoot, err) } @@ -212,28 +213,53 @@ func partitionBluestoreMetadata(context *clusterd.Context, info *partition.Metad return nil } -// Partitions a device for use by a bluestore osd. +// Partitions a device for use by a osd. // If there are any partitions or formatting already on the device, it will be wiped. -func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error { - blockDetails, err := getBlockPartitionDetails(config) +func partitionOSD(context *clusterd.Context, config *osdConfig) error { + dataDetails, err := getDataPartitionDetails(config) if err != nil { return err } // zap/clear all existing partitions on the device - err = sys.RemovePartitions(blockDetails.Device, context.Executor) + err = sys.RemovePartitions(dataDetails.Device, context.Executor) if err != nil { - return fmt.Errorf("failed to zap partitions on metadata device /dev/%s: %+v", blockDetails.Device, err) + return fmt.Errorf("failed to zap partitions on metadata device /dev/%s: %+v", dataDetails.Device, err) } // create the partitions on the device - err = sys.CreatePartitions(blockDetails.Device, config.partitionScheme.GetPartitionArgs(), context.Executor) + err = sys.CreatePartitions(dataDetails.Device, config.partitionScheme.GetPartitionArgs(), context.Executor) if err != nil { - return fmt.Errorf("failed to partition /dev/%s. %+v", blockDetails.Device, err) + return fmt.Errorf("failed to partition /dev/%s. %+v", dataDetails.Device, err) + } + + if config.partitionScheme.StoreType == Filestore { + // the OSD is using filestore, create a filesystem for the device and mount it under config root + dataPartDetails := config.partitionScheme.Partitions[FilestoreDataPartitionType] + dataPartPath := filepath.Join(diskByPartUUID, dataPartDetails.PartitionUUID) + logger.Infof("waiting for partition path %s", dataPartPath) + err = waitForPath(dataPartPath, context.Executor) + if err != nil { + return fmt.Errorf("failed waiting for %s: %+v", dataPartPath, err) + } + err = sys.FormatDevice(dataPartPath, context.Executor) + if err != nil { + logger.Warningf("first attempt to format partition %s on device %s failed. Waiting 2 seconds then retrying: %+v", + dataPartDetails.PartitionUUID, dataPartDetails.Device, err) + <-time.After(2 * time.Second) + err = sys.FormatDevice(dataPartPath, context.Executor) + if err != nil { + return fmt.Errorf("failed to format partition %s on device %s. %+v", dataPartDetails.PartitionUUID, dataPartDetails.Device, err) + } + } + err = sys.MountDevice(dataPartPath, config.rootPath, context.Executor) + if err != nil { + return fmt.Errorf("failed to mount %s at %s: %+v", dataPartPath, config.configRoot, context.Executor) + } } // save the partition scheme entry to disk now that it has been committed - savedScheme, err := partition.LoadScheme(config.configRoot) + savedScheme, err := LoadScheme(config.configRoot) if err != nil { return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", config.configRoot, err) } @@ -243,27 +269,27 @@ func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error { } // update the uuid of the disk in the inventory in memory - logger.Debugf("Updating disk uuid %s on device %s", blockDetails.DiskUUID, blockDetails.Device) + logger.Debugf("Updating disk uuid %s on device %s", dataDetails.DiskUUID, dataDetails.Device) for _, disk := range context.Inventory.Local.Disks { - if disk.Name == blockDetails.Device { - logger.Debugf("Updated uuid on device %s", blockDetails.Device) - disk.UUID = blockDetails.DiskUUID + if disk.Name == dataDetails.Device { + logger.Debugf("Updated uuid on device %s", dataDetails.Device) + disk.UUID = dataDetails.DiskUUID } } // save the desired state of the osd for this device if context.EtcdClient != nil { - err = associateOsdIDWithDevice(context.EtcdClient, context.NodeID, blockDetails.DiskUUID, config.id, false) + err = associateOsdIDWithDevice(context.EtcdClient, context.NodeID, dataDetails.DiskUUID, config.id, false) if err != nil { - return fmt.Errorf("failed to associate osd id %d with device %s (%s)", config.id, blockDetails.Device, blockDetails.DiskUUID) + return fmt.Errorf("failed to associate osd id %d with device %s (%s)", config.id, dataDetails.Device, dataDetails.DiskUUID) } if config.partitionScheme.IsCollocated() { // the metadata is on the same disk as the data, associate the osd ID with the device for metadata too err = associateOSDIDsWithMetadataDevice( - context.EtcdClient, context.NodeID, blockDetails.DiskUUID, fmt.Sprintf("%d", config.id)) + context.EtcdClient, context.NodeID, dataDetails.DiskUUID, fmt.Sprintf("%d", config.id)) if err != nil { return fmt.Errorf("failed to associate osd id %d with device %s (%s) for metadata", - config.id, blockDetails.Device, blockDetails.DiskUUID) + config.id, dataDetails.Device, dataDetails.DiskUUID) } } } @@ -271,17 +297,39 @@ func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error { return nil } -func getBlockPartitionDetails(config *osdConfig) (*partition.PerfSchemePartitionDetails, error) { +func getDataPartitionDetails(config *osdConfig) (*PerfSchemePartitionDetails, error) { if config.partitionScheme == nil { return nil, fmt.Errorf("partition scheme missing from %+v", config) } - blockDetails, ok := config.partitionScheme.Partitions[partition.BlockPartitionName] - if !ok || blockDetails == nil { - return nil, fmt.Errorf("block partition missing from %+v", config.partitionScheme) + dataPartitionType := config.partitionScheme.getDataPartitionType() + + dataDetails, ok := config.partitionScheme.Partitions[dataPartitionType] + if !ok || dataDetails == nil { + return nil, fmt.Errorf("data partition missing from %+v", config.partitionScheme) } - return blockDetails, nil + return dataDetails, nil +} + +func getMetadataPartitionDetails(config *osdConfig) (*PerfSchemePartitionDetails, error) { + if config.partitionScheme == nil { + return nil, fmt.Errorf("partition scheme missing from %+v", config) + } + + metadataPartitionType := config.partitionScheme.getMetadataPartitionType() + + if config.partitionScheme.StoreType == Filestore { + // TODO: support separate metadata device for filestore (just use the data partition details for now) + return getDataPartitionDetails(config) + } + + metadataDetails, ok := config.partitionScheme.Partitions[metadataPartitionType] + if !ok || metadataDetails == nil { + return nil, fmt.Errorf("metadata partition missing from %+v", config.partitionScheme) + } + + return metadataDetails, nil } func getDiskSize(context *clusterd.Context, name string) (uint64, error) { @@ -294,6 +342,24 @@ func getDiskSize(context *clusterd.Context, name string) (uint64, error) { return 0, fmt.Errorf("device %s not found", name) } +func waitForPath(path string, executor exec.Executor) error { + retryCount := 0 + retryMax := 25 + sleepTime := 250 + for { + _, err := executor.ExecuteStat(path) + if err == nil { + return nil + } + + retryCount++ + if retryCount > retryMax { + return err + } + <-time.After(time.Duration(sleepTime) * time.Millisecond) + } +} + func registerOSD(bootstrapConn client.Connection) (*int, *uuid.UUID, error) { var err error osdUUID, err := uuid.NewRandom() @@ -311,9 +377,15 @@ func registerOSD(bootstrapConn client.Connection) (*int, *uuid.UUID, error) { return &osdID, &osdUUID, nil } -func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]string, error) { +func getStoreSettings(context *clusterd.Context, config *osdConfig, storeConfig StoreConfig) (map[string]string, error) { settings := map[string]string{} - if config.dir { + if config.dir || (config.partitionScheme != nil && config.partitionScheme.StoreType == Filestore) { + // add additional filestore settings for dirs and filestore devices + journalSize := JournalDefaultSizeMB + if storeConfig.JournalSizeMB > 0 { + journalSize = storeConfig.JournalSizeMB + } + settings["osd journal size"] = strconv.Itoa(journalSize) return settings, nil } @@ -321,21 +393,22 @@ func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string] return nil, fmt.Errorf("failed to find partitions from config for osd %d", config.id) } + // add additional bluestore settings parts := config.partitionScheme.Partitions - walPartition, ok := parts[partition.WalPartitionName] + walPartition, ok := parts[WalPartitionType] if !ok { return nil, fmt.Errorf("failed to find wal partition for osd %d", config.id) } - dbPartition, ok := parts[partition.DatabasePartitionName] + dbPartition, ok := parts[DatabasePartitionType] if !ok { return nil, fmt.Errorf("failed to find db partition for osd %d", config.id) } - blockPartition, ok := parts[partition.BlockPartitionName] + blockPartition, ok := parts[BlockPartitionType] if !ok { return nil, fmt.Errorf("failed to find block partition for osd %d", config.id) } - prefix := "/dev/disk/by-partuuid" + prefix := diskByPartUUID settings["bluestore block wal path"] = path.Join(prefix, walPartition.PartitionUUID) settings["bluestore block db path"] = path.Join(prefix, dbPartition.PartitionUUID) settings["bluestore block path"] = path.Join(prefix, blockPartition.PartitionUUID) @@ -343,12 +416,12 @@ func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string] return settings, nil } -func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context, - bootstrapConn client.Connection, cluster *mon.ClusterInfo, location string, executor exec.Executor) error { +func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context, bootstrapConn client.Connection, + cluster *mon.ClusterInfo, location string, storeConfig StoreConfig, executor exec.Executor) error { - cephConfig := mon.CreateDefaultCephConfig(context, cluster, config.rootPath, !config.dir) + cephConfig := mon.CreateDefaultCephConfig(context, cluster, config.rootPath, isBluestore(config)) - if config.dir { + if !isBluestore(config) { // using the local file system requires some config overrides // http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/#not-recommended cephConfig.GlobalConfig.OsdMaxObjectNameLen = 256 @@ -356,7 +429,7 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context } // bluestore has some extra settings - settings, err := getStoreSettings(context, config) + settings, err := getStoreSettings(context, config, storeConfig) if err != nil { return fmt.Errorf("failed to read store settings. %+v", err) } @@ -364,7 +437,7 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context // write the OSD config file to disk keyringPath := getOSDKeyringPath(config.rootPath) _, err = mon.GenerateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id), - keyringPath, !config.dir, cephConfig, settings) + keyringPath, isBluestore(config), cephConfig, settings) if err != nil { return fmt.Errorf("failed to write OSD %d config file: %+v", config.id, err) } @@ -461,7 +534,7 @@ func createOSDFileSystem(context *clusterd.Context, clusterName string, config * fmt.Sprintf("--monmap=%s", monMapTmpPath), } - if config.dir { + if !isBluestore(config) { options = append(options, fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(config.rootPath))) options = append(options, fmt.Sprintf("--keyring=%s", getOSDKeyringPath(config.rootPath))) } @@ -523,8 +596,6 @@ func addOSDToCrushMap(osdConn client.Connection, context *clusterd.Context, osdI weight, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", weight), 64) osdEntity := fmt.Sprintf("osd.%d", osdID) - logger.Infof("OSD %s at %s, bytes: %d, weight: %.4f", osdEntity, osdDataPath, all, weight) - locArgs, err := formatLocation(location) if err != nil { return err @@ -536,6 +607,10 @@ func addOSDToCrushMap(osdConn client.Connection, context *clusterd.Context, osdI "weight": weight, "args": locArgs, } + + logger.Infof("adding %s (%s), bytes: %d, weight: %.4f, to crush map at '%s'", + osdEntity, osdDataPath, all, weight, strings.Join(locArgs, " ")) + _, err = client.ExecuteMonCommand(osdConn, cmd, fmt.Sprintf("adding %s to crush map", osdEntity)) if err != nil { return fmt.Errorf("failed adding %s to crush map: %+v", osdEntity, err) diff --git a/pkg/cephmgr/osd/device_test.go b/pkg/cephmgr/osd/device_test.go index 139dd1156..0b911f50d 100644 --- a/pkg/cephmgr/osd/device_test.go +++ b/pkg/cephmgr/osd/device_test.go @@ -25,7 +25,6 @@ import ( "github.com/google/uuid" testceph "github.com/rook/rook/pkg/cephmgr/client/test" - "github.com/rook/rook/pkg/cephmgr/osd/partition" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/clusterd/inventory" "github.com/rook/rook/pkg/util" @@ -133,11 +132,13 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda" PARTLABEL="ROOK-OSD0-BLOCK"` return output, nil } + storeConfig := StoreConfig{StoreType: Bluestore} + // set up a partition scheme entry for sda (collocated metadata and data) - entry := partition.NewPerfSchemeEntry() + entry := NewPerfSchemeEntry(storeConfig.StoreType) entry.ID = 1 entry.OsdUUID = uuid.Must(uuid.NewRandom()) - partition.PopulateCollocatedPerfSchemeEntry(entry, "sda", partition.BluestoreConfig{}) + PopulateCollocatedPerfSchemeEntry(entry, "sda", storeConfig) context := &clusterd.Context{EtcdClient: etcdClient, Executor: executor, NodeID: nodeID, ConfigDir: configDir, Inventory: createInventory()} @@ -154,7 +155,7 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda" PARTLABEL="ROOK-OSD0-BLOCK"` // try to format the device. even though the device has existing partitions, they are owned by rook, so it is safe // to format and the format/partitioning will happen. - err = formatDevice(context, config, false) + err = formatDevice(context, config, false, storeConfig) assert.Nil(t, err) assert.Equal(t, 3, execCount) assert.Equal(t, 3, outputExecCount) @@ -195,21 +196,21 @@ func TestPartitionBluestoreMetadata(t *testing.T) { context := &clusterd.Context{EtcdClient: etcdClient, Executor: executor, NodeID: nodeID, ConfigDir: configDir} // create metadata partition information for 2 OSDs (sdb, sdc) storing their metadata on device sda - bluestoreConfig := partition.BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2} - metadata := partition.NewMetadataDeviceInfo("sda") + storeConfig := StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2} + metadata := NewMetadataDeviceInfo("sda") - e1 := partition.NewPerfSchemeEntry() + e1 := NewPerfSchemeEntry(Bluestore) e1.ID = 1 e1.OsdUUID = uuid.Must(uuid.NewRandom()) - partition.PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, bluestoreConfig) + PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, storeConfig) - e2 := partition.NewPerfSchemeEntry() + e2 := NewPerfSchemeEntry(Bluestore) e2.ID = 2 e2.OsdUUID = uuid.Must(uuid.NewRandom()) - partition.PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, bluestoreConfig) + PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, storeConfig) // perform the metadata device partition - err = partitionBluestoreMetadata(context, metadata, configDir) + err = partitionMetadata(context, metadata, configDir) assert.Nil(t, err) assert.Equal(t, 3, execCount) @@ -221,7 +222,12 @@ func TestPartitionBluestoreMetadata(t *testing.T) { assert.True(t, util.CreateSet(desiredIds).Equals(util.CreateSet([]string{"1", "2"}))) } -func TestPartitionBluestoreOSD(t *testing.T) { +func TestPartitionOSD(t *testing.T) { + testPartitionOSDHelper(t, StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2}) + testPartitionOSDHelper(t, StoreConfig{StoreType: Filestore}) +} + +func testPartitionOSDHelper(t *testing.T, storeConfig StoreConfig) { // set up a temporary config directory that will be cleaned up after test configDir, err := ioutil.TempDir("", "TestPartitionBluestoreOSD") if err != nil { @@ -237,17 +243,36 @@ func TestPartitionBluestoreOSD(t *testing.T) { executor := &exectest.MockExecutor{} executor.MockExecuteCommand = func(name string, command string, args ...string) error { logger.Infof("RUN %d for '%s'. %s %+v", execCount, name, command, args) - assert.Equal(t, "sgdisk", command) + if execCount <= 2 { + assert.Equal(t, "sgdisk", command) + } switch execCount { case 0: assert.Equal(t, []string{"--zap-all", "/dev/sda"}, args) case 1: assert.Equal(t, []string{"--clear", "--mbrtogpt", "/dev/sda"}, args) case 2: - assert.Equal(t, 11, len(args)) - assert.Equal(t, "--change-name=1:ROOK-OSD1-WAL", args[1]) - assert.Equal(t, "--change-name=2:ROOK-OSD1-DB", args[4]) - assert.Equal(t, "--change-name=3:ROOK-OSD1-BLOCK", args[7]) + if storeConfig.StoreType == Bluestore { + assert.Equal(t, 11, len(args)) + assert.Equal(t, "--change-name=1:ROOK-OSD1-WAL", args[1]) + assert.Equal(t, "--change-name=2:ROOK-OSD1-DB", args[4]) + assert.Equal(t, "--change-name=3:ROOK-OSD1-BLOCK", args[7]) + } else { + assert.Equal(t, 5, len(args)) + assert.Equal(t, "--change-name=1:ROOK-OSD1-FS-DATA", args[1]) + } + case 3: + if storeConfig.StoreType == Bluestore { + assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount)) + } else { + assert.Equal(t, "mkfs.ext4", command) + } + case 4: + if storeConfig.StoreType == Bluestore { + assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount)) + } else { + assert.Equal(t, "mount", command) + } } execCount++ return nil @@ -260,28 +285,32 @@ func TestPartitionBluestoreOSD(t *testing.T) { } // setup a partition scheme for data and metadata to be collocated on sda - bluestoreConfig := partition.BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2} - entry := partition.NewPerfSchemeEntry() + entry := NewPerfSchemeEntry(storeConfig.StoreType) entry.ID = 1 entry.OsdUUID = uuid.Must(uuid.NewRandom()) - partition.PopulateCollocatedPerfSchemeEntry(entry, "sda", bluestoreConfig) + PopulateCollocatedPerfSchemeEntry(entry, "sda", storeConfig) config := &osdConfig{configRoot: configDir, rootPath: filepath.Join(configDir, "osd1"), id: entry.ID, uuid: entry.OsdUUID, dir: false, partitionScheme: entry} // partition the OSD on sda now - err = partitionBluestoreOSD(context, config) + err = partitionOSD(context, config) assert.Nil(t, err) - assert.Equal(t, 3, execCount) + + if storeConfig.StoreType == Bluestore { + assert.Equal(t, 3, execCount) + } else { + assert.Equal(t, 5, execCount) + } // verify that both the data and metadata have been associated with the device in etcd (since data/metadata are collocated) - blockDetails, err := getBlockPartitionDetails(config) + dataDetails, err := getDataPartitionDetails(config) assert.Nil(t, err) dataID := etcdClient.GetValue( - fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-data", blockDetails.DiskUUID)) + fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-data", dataDetails.DiskUUID)) assert.Equal(t, "1", dataID) metadataID := etcdClient.GetValue( - fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-metadata", blockDetails.DiskUUID)) + fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-metadata", dataDetails.DiskUUID)) assert.Equal(t, "1", metadataID) } diff --git a/pkg/cephmgr/osd/partition/scheme.go b/pkg/cephmgr/osd/scheme.go similarity index 57% rename from pkg/cephmgr/osd/partition/scheme.go rename to pkg/cephmgr/osd/scheme.go index 66d3fd23e..ae40614d2 100644 --- a/pkg/cephmgr/osd/partition/scheme.go +++ b/pkg/cephmgr/osd/scheme.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package partition +package osd import ( "encoding/json" @@ -26,21 +26,31 @@ import ( ) const ( - WalPartitionName = "wal" - DatabasePartitionName = "db" - BlockPartitionName = "block" - walPartition = 1 - databasePartition = 2 - blockPartition = 3 - UseRemainingSpace = -1 - schemeFilename = "partition-scheme" - WalDefaultSizeMB = 576 - DBDefaultSizeMB = 20480 + Filestore = "filestore" + Bluestore = "bluestore" + DefaultStore = Filestore + UseRemainingSpace = -1 + schemeFilename = "partition-scheme" + WalDefaultSizeMB = 576 + DBDefaultSizeMB = 20480 + JournalDefaultSizeMB = 5120 ) -type BluestoreConfig struct { - WalSizeMB int - DatabaseSizeMB int +type PartitionType int + +const ( + WalPartitionType PartitionType = iota + DatabasePartitionType + BlockPartitionType + FilestoreDataPartitionType + FilestoreJournalPartitionType +) + +type StoreConfig struct { + StoreType string `json:"storeType,omitempty"` + WalSizeMB int `json:"walSizeMB,omitempty"` + DatabaseSizeMB int `json:"databaseSizeMB,omitempty"` + JournalSizeMB int `json:"journalSizeMB,omitempty"` } // top level representation of an overall performance oriented partition scheme, with a dedicated metadata device @@ -52,9 +62,10 @@ type PerfScheme struct { // represents an OSD and details about all of its partitions type PerfSchemeEntry struct { - ID int `json:"id"` - OsdUUID uuid.UUID `json:"osdUuid"` - Partitions map[string]*PerfSchemePartitionDetails `json:"partitions"` // mapping of partition name to its details + ID int `json:"id"` + OsdUUID uuid.UUID `json:"osdUuid"` + Partitions map[PartitionType]*PerfSchemePartitionDetails `json:"partitions"` // mapping of partition name to its details + StoreType string `json:"storeType,omitempty"` } // details for 1 OSD partition @@ -75,12 +86,12 @@ type MetadataDeviceInfo struct { // representsa specific partition on a metadata device, including details about which OSD it belongs to type MetadataDevicePartition struct { - ID int `json:"id"` - OsdUUID uuid.UUID `json:"osdUuid"` - Name string `json:"name"` - PartitionUUID string `json:"partitionUuid"` - SizeMB int `json:"sizeMB"` - OffsetMB int `json:"offsetMB"` + ID int `json:"id"` + OsdUUID uuid.UUID `json:"osdUuid"` + Type PartitionType `json:"type"` + PartitionUUID string `json:"partitionUuid"` + SizeMB int `json:"sizeMB"` + OffsetMB int `json:"offsetMB"` } func NewPerfScheme() *PerfScheme { @@ -89,9 +100,10 @@ func NewPerfScheme() *PerfScheme { } } -func NewPerfSchemeEntry() *PerfSchemeEntry { +func NewPerfSchemeEntry(storeType string) *PerfSchemeEntry { return &PerfSchemeEntry{ - Partitions: map[string]*PerfSchemePartitionDetails{}, // mapping of partition name (e.g. WAL) to it's details + Partitions: map[PartitionType]*PerfSchemePartitionDetails{}, // mapping of partition name (e.g. WAL) to it's details + StoreType: storeType, } } @@ -137,48 +149,66 @@ func (s *PerfScheme) Save(configDir string) error { } // populates a partition scheme entry for an OSD where all its partitions are collocated on a single device -func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, bluestoreConfig BluestoreConfig) error { - diskUUID, walUUID, dbUUID, blockUUID, err := createUUIDs() - if err != nil { - return err - } +func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, storeConfig StoreConfig) error { - walSize := WalDefaultSizeMB - if bluestoreConfig.WalSizeMB > 0 { - walSize = bluestoreConfig.WalSizeMB - } - dbSize := DBDefaultSizeMB - if bluestoreConfig.DatabaseSizeMB > 0 { - dbSize = bluestoreConfig.DatabaseSizeMB - } + if storeConfig.StoreType == Filestore { + diskUUID, dataUUID, _, err := createFilestoreUUIDs() + if err != nil { + return err + } - offset := 1 + // the filestore data partition will take up the entire given device (and we do not create a separate partition/entry + // for the journal) + entry.Partitions[FilestoreDataPartitionType] = &PerfSchemePartitionDetails{ + Device: device, + DiskUUID: diskUUID.String(), + PartitionUUID: dataUUID.String(), + SizeMB: UseRemainingSpace, + OffsetMB: 1, + } + } else { + diskUUID, walUUID, dbUUID, blockUUID, err := createBluestoreUUIDs() + if err != nil { + return err + } - // layout the partitions for WAL, DB, and Block - entry.Partitions[WalPartitionName] = &PerfSchemePartitionDetails{ - Device: device, - DiskUUID: diskUUID.String(), - PartitionUUID: walUUID.String(), - SizeMB: walSize, - OffsetMB: offset, - } - offset += entry.Partitions[WalPartitionName].SizeMB + walSize := WalDefaultSizeMB + if storeConfig.WalSizeMB > 0 { + walSize = storeConfig.WalSizeMB + } + dbSize := DBDefaultSizeMB + if storeConfig.DatabaseSizeMB > 0 { + dbSize = storeConfig.DatabaseSizeMB + } - entry.Partitions[DatabasePartitionName] = &PerfSchemePartitionDetails{ - Device: device, - DiskUUID: diskUUID.String(), - PartitionUUID: dbUUID.String(), - SizeMB: dbSize, - OffsetMB: offset, - } - offset += entry.Partitions[DatabasePartitionName].SizeMB + offset := 1 - entry.Partitions[BlockPartitionName] = &PerfSchemePartitionDetails{ - Device: device, - DiskUUID: diskUUID.String(), - PartitionUUID: blockUUID.String(), - SizeMB: UseRemainingSpace, - OffsetMB: offset, + // layout the partitions for WAL, DB, and Block + entry.Partitions[WalPartitionType] = &PerfSchemePartitionDetails{ + Device: device, + DiskUUID: diskUUID.String(), + PartitionUUID: walUUID.String(), + SizeMB: walSize, + OffsetMB: offset, + } + offset += entry.Partitions[WalPartitionType].SizeMB + + entry.Partitions[DatabasePartitionType] = &PerfSchemePartitionDetails{ + Device: device, + DiskUUID: diskUUID.String(), + PartitionUUID: dbUUID.String(), + SizeMB: dbSize, + OffsetMB: offset, + } + offset += entry.Partitions[DatabasePartitionType].SizeMB + + entry.Partitions[BlockPartitionType] = &PerfSchemePartitionDetails{ + Device: device, + DiskUUID: diskUUID.String(), + PartitionUUID: blockUUID.String(), + SizeMB: UseRemainingSpace, + OffsetMB: offset, + } } return nil @@ -187,15 +217,20 @@ func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, bl // populates a partition scheme entry for an OSD that will have distributed partitions: its metadata will live on a // dedicated metadata device and its block data will live on a dedicated device func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, metadataInfo *MetadataDeviceInfo, - bluestoreConfig BluestoreConfig) error { + storeConfig StoreConfig) error { - diskUUID, walUUID, dbUUID, blockUUID, err := createUUIDs() + if storeConfig.StoreType == Filestore { + // TODO: support separate metadata device for filestore + return fmt.Errorf("filestore not yet supported for distributed partition scheme") + } + + diskUUID, walUUID, dbUUID, blockUUID, err := createBluestoreUUIDs() if err != nil { return err } // the block partition will take up the entire given device - entry.Partitions[BlockPartitionName] = &PerfSchemePartitionDetails{ + entry.Partitions[BlockPartitionType] = &PerfSchemePartitionDetails{ Device: device, DiskUUID: diskUUID.String(), PartitionUUID: blockUUID.String(), @@ -219,16 +254,16 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m } walSize := WalDefaultSizeMB - if bluestoreConfig.WalSizeMB > 0 { - walSize = bluestoreConfig.WalSizeMB + if storeConfig.WalSizeMB > 0 { + walSize = storeConfig.WalSizeMB } dbSize := DBDefaultSizeMB - if bluestoreConfig.DatabaseSizeMB > 0 { - dbSize = bluestoreConfig.DatabaseSizeMB + if storeConfig.DatabaseSizeMB > 0 { + dbSize = storeConfig.DatabaseSizeMB } // record information about the WAL partition - entry.Partitions[WalPartitionName] = &PerfSchemePartitionDetails{ + entry.Partitions[WalPartitionType] = &PerfSchemePartitionDetails{ Device: metadataInfo.Device, DiskUUID: metadataInfo.DiskUUID, PartitionUUID: walUUID.String(), @@ -238,16 +273,16 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m walPartitionInfo := &MetadataDevicePartition{ ID: entry.ID, OsdUUID: entry.OsdUUID, - Name: WalPartitionName, + Type: WalPartitionType, PartitionUUID: walUUID.String(), SizeMB: walSize, OffsetMB: offset, } metadataInfo.Partitions = append(metadataInfo.Partitions, walPartitionInfo) - offset += entry.Partitions[WalPartitionName].SizeMB + offset += entry.Partitions[WalPartitionType].SizeMB // record information about the DB partition - entry.Partitions[DatabasePartitionName] = &PerfSchemePartitionDetails{ + entry.Partitions[DatabasePartitionType] = &PerfSchemePartitionDetails{ Device: metadataInfo.Device, DiskUUID: metadataInfo.DiskUUID, PartitionUUID: dbUUID.String(), @@ -257,7 +292,7 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m dbPartitionInfo := &MetadataDevicePartition{ ID: entry.ID, OsdUUID: entry.OsdUUID, - Name: DatabasePartitionName, + Type: DatabasePartitionType, PartitionUUID: dbUUID.String(), SizeMB: dbSize, OffsetMB: offset, @@ -271,7 +306,7 @@ func (m *MetadataDeviceInfo) GetPartitionArgs() []string { args := []string{} for partNum, part := range m.Partitions { - partArgs := getPartitionArgs(partNum+1, part.PartitionUUID, part.OffsetMB, part.SizeMB, getPartitionLabel(part.ID, part.Name)) + partArgs := getPartitionArgs(partNum+1, part.PartitionUUID, part.OffsetMB, part.SizeMB, getPartitionLabel(part.ID, part.Type)) args = append(args, partArgs...) } @@ -288,26 +323,28 @@ func (e *PerfSchemeEntry) GetPartitionArgs() []string { args := []string{} partNum := 1 - if collocated { + if collocated && e.StoreType == Bluestore { // partitions are collocated, create the metadata partitions on the same device - walDetails := e.Partitions[WalPartitionName] - partArgs := getPartitionArgsFromDetails(partNum, WalPartitionName, e.ID, walDetails) + walDetails := e.Partitions[WalPartitionType] + partArgs := getPartitionArgsFromDetails(partNum, WalPartitionType, e.ID, walDetails) args = append(args, partArgs...) partNum++ - dbDetails := e.Partitions[DatabasePartitionName] - partArgs = getPartitionArgsFromDetails(partNum, DatabasePartitionName, e.ID, dbDetails) + dbDetails := e.Partitions[DatabasePartitionType] + partArgs = getPartitionArgsFromDetails(partNum, DatabasePartitionType, e.ID, dbDetails) args = append(args, partArgs...) partNum++ } + dataPartitionType := e.getDataPartitionType() + // always create the data partition - blockDetails := e.Partitions[BlockPartitionName] - partArgs := getPartitionArgsFromDetails(partNum, BlockPartitionName, e.ID, blockDetails) - args = append(args, partArgs...) + dataDetails := e.Partitions[dataPartitionType] + dataPartArgs := getPartitionArgsFromDetails(partNum, dataPartitionType, e.ID, dataDetails) + args = append(args, dataPartArgs...) // append args for the whole device - args = append(args, []string{fmt.Sprintf("--disk-guid=%s", blockDetails.DiskUUID), "/dev/" + blockDetails.Device}...) + args = append(args, []string{fmt.Sprintf("--disk-guid=%s", dataDetails.DiskUUID), "/dev/" + dataDetails.Device}...) return args } @@ -329,10 +366,26 @@ func (e *PerfSchemeEntry) IsCollocated() bool { return collocated } +func (e *PerfSchemeEntry) getDataPartitionType() PartitionType { + if e.StoreType == Filestore { + return FilestoreDataPartitionType + } else { + return BlockPartitionType + } +} + +func (e *PerfSchemeEntry) getMetadataPartitionType() PartitionType { + if e.StoreType == Filestore { + return FilestoreJournalPartitionType + } else { + return DatabasePartitionType + } +} + // Get the arguments necessary to create an sgdisk partition with the given parameters. // number is the partition number. // The offset and length are in MB. Under the covers this is translated to sectors. -// If the length is -1, all remaining space will be assigned to the partition. +// If the length is -1, all remaining space will be assigned to the func getPartitionArgs(number int, guid string, offset, length int, label string) []string { const sectorsPerMB = 2048 var newPart string @@ -351,24 +404,28 @@ func getPartitionArgs(number int, guid string, offset, length int, label string) } } -func getPartitionArgsFromDetails(number int, name string, id int, details *PerfSchemePartitionDetails) []string { - return getPartitionArgs(number, details.PartitionUUID, details.OffsetMB, details.SizeMB, getPartitionLabel(id, name)) +func getPartitionArgsFromDetails(number int, partType PartitionType, id int, details *PerfSchemePartitionDetails) []string { + return getPartitionArgs(number, details.PartitionUUID, details.OffsetMB, details.SizeMB, getPartitionLabel(id, partType)) } -func getPartitionLabel(id int, partName string) string { - switch partName { - case WalPartitionName: +func getPartitionLabel(id int, partType PartitionType) string { + switch partType { + case WalPartitionType: return fmt.Sprintf("ROOK-OSD%d-WAL", id) - case DatabasePartitionName: + case DatabasePartitionType: return fmt.Sprintf("ROOK-OSD%d-DB", id) - case BlockPartitionName: + case BlockPartitionType: return fmt.Sprintf("ROOK-OSD%d-BLOCK", id) + case FilestoreDataPartitionType: + return fmt.Sprintf("ROOK-OSD%d-FS-DATA", id) + case FilestoreJournalPartitionType: + return fmt.Sprintf("ROOK-OSD%d-FS-JOURNAL", id) } return "" } -func createUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) { +func createBluestoreUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) { diskUUID, err := uuid.NewRandom() if err != nil { return nil, nil, nil, nil, fmt.Errorf("failed to get disk uuid. %+v", err) @@ -391,3 +448,22 @@ func createUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) { return &diskUUID, &walUUID, &dbUUID, &blockUUID, nil } + +func createFilestoreUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, error) { + diskUUID, err := uuid.NewRandom() + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get disk uuid. %+v", err) + } + + dataUUID, err := uuid.NewRandom() + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get data uuid. %+v", err) + } + + journalUUID, err := uuid.NewRandom() + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get journal uuid. %+v", err) + } + + return &diskUUID, &dataUUID, &journalUUID, nil +} diff --git a/pkg/cephmgr/osd/partition/scheme_test.go b/pkg/cephmgr/osd/scheme_test.go similarity index 75% rename from pkg/cephmgr/osd/partition/scheme_test.go rename to pkg/cephmgr/osd/scheme_test.go index 99285f9bc..f672da377 100644 --- a/pkg/cephmgr/osd/partition/scheme_test.go +++ b/pkg/cephmgr/osd/scheme_test.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package partition +package osd import ( "fmt" @@ -43,15 +43,15 @@ func TestSchemeSaveLoad(t *testing.T) { // add some entries to the scheme scheme.Metadata = NewMetadataDeviceInfo("sda") scheme.Metadata.DiskUUID = uuid.Must(uuid.NewRandom()).String() - m1 := &MetadataDevicePartition{ID: 1, OsdUUID: uuid.Must(uuid.NewRandom()), Name: "wal", + m1 := &MetadataDevicePartition{ID: 1, OsdUUID: uuid.Must(uuid.NewRandom()), Type: WalPartitionType, PartitionUUID: uuid.Must(uuid.NewRandom()).String(), SizeMB: 100, OffsetMB: 1} - m2 := &MetadataDevicePartition{ID: 1, OsdUUID: m1.OsdUUID, Name: "db", + m2 := &MetadataDevicePartition{ID: 1, OsdUUID: m1.OsdUUID, Type: DatabasePartitionType, PartitionUUID: uuid.Must(uuid.NewRandom()).String(), SizeMB: 200, OffsetMB: 101} scheme.Metadata.Partitions = append(scheme.Metadata.Partitions, []*MetadataDevicePartition{m1, m2}...) e1 := &PerfSchemeEntry{ID: 1, OsdUUID: m1.OsdUUID} - e1.Partitions = map[string]*PerfSchemePartitionDetails{ - "block": &PerfSchemePartitionDetails{ + e1.Partitions = map[PartitionType]*PerfSchemePartitionDetails{ + BlockPartitionType: &PerfSchemePartitionDetails{ Device: "sdb", DiskUUID: uuid.Must(uuid.NewRandom()).String(), PartitionUUID: uuid.Must(uuid.NewRandom()).String(), @@ -71,45 +71,45 @@ func TestSchemeSaveLoad(t *testing.T) { } func TestPopulateCollocatedPerfSchemeEntry(t *testing.T) { - entry := NewPerfSchemeEntry() + entry := NewPerfSchemeEntry(Bluestore) entry.ID = 10 entry.OsdUUID = uuid.Must(uuid.NewRandom()) - err := PopulateCollocatedPerfSchemeEntry(entry, "sda", BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}) + err := PopulateCollocatedPerfSchemeEntry(entry, "sda", StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}) assert.Nil(t, err) // verify the populated collocated partition entries assert.Equal(t, 3, len(entry.Partitions)) - verifyPartitionDetails(t, entry, WalPartitionName, "sda", 1, 1) - verifyPartitionDetails(t, entry, DatabasePartitionName, "sda", 2, 2) - verifyPartitionDetails(t, entry, BlockPartitionName, "sda", 4, -1) + verifyPartitionDetails(t, entry, WalPartitionType, "sda", 1, 1) + verifyPartitionDetails(t, entry, DatabasePartitionType, "sda", 2, 2) + verifyPartitionDetails(t, entry, BlockPartitionType, "sda", 4, -1) } func TestPopulateDistributedPerfSchemeEntry(t *testing.T) { metadata := NewMetadataDeviceInfo("sda") - entry := NewPerfSchemeEntry() + entry := NewPerfSchemeEntry(Bluestore) entry.ID = 20 entry.OsdUUID = uuid.Must(uuid.NewRandom()) - err := PopulateDistributedPerfSchemeEntry(entry, "sdb", metadata, BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}) + err := PopulateDistributedPerfSchemeEntry(entry, "sdb", metadata, StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}) assert.Nil(t, err) // verify the populated distributed partition entries (metadata partitions should be on metadata device, block // partition should be on its own device) assert.Equal(t, 3, len(entry.Partitions)) - verifyPartitionDetails(t, entry, WalPartitionName, "sda", 1, 1) - verifyPartitionDetails(t, entry, DatabasePartitionName, "sda", 2, 2) - verifyPartitionDetails(t, entry, BlockPartitionName, "sdb", 1, -1) + verifyPartitionDetails(t, entry, WalPartitionType, "sda", 1, 1) + verifyPartitionDetails(t, entry, DatabasePartitionType, "sda", 2, 2) + verifyPartitionDetails(t, entry, BlockPartitionType, "sdb", 1, -1) // verify that the metadata device info was populated as well assert.Equal(t, 2, len(metadata.Partitions)) - verifyMetadataDevicePartition(t, metadata, 0, entry.ID, entry.OsdUUID, WalPartitionName, 1, 1) - verifyMetadataDevicePartition(t, metadata, 1, entry.ID, entry.OsdUUID, DatabasePartitionName, 2, 2) + verifyMetadataDevicePartition(t, metadata, 0, entry.ID, entry.OsdUUID, WalPartitionType, 1, 1) + verifyMetadataDevicePartition(t, metadata, 1, entry.ID, entry.OsdUUID, DatabasePartitionType, 2, 2) } -func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partName, device string, offset, size int64) { - part, ok := entry.Partitions[partName] +func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partType PartitionType, device string, offset, size int64) { + part, ok := entry.Partitions[partType] assert.True(t, ok) assert.NotNil(t, part) assert.Equal(t, device, part.Device) @@ -118,13 +118,13 @@ func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partName, devi } func verifyMetadataDevicePartition(t *testing.T, info *MetadataDeviceInfo, index int, osdID int, osdUUID uuid.UUID, - name string, offset, size int64) { + partType PartitionType, offset, size int64) { part := info.Partitions[index] assert.NotNil(t, part) assert.Equal(t, osdID, part.ID) assert.Equal(t, osdUUID, part.OsdUUID) - assert.Equal(t, name, part.Name) + assert.Equal(t, partType, part.Type) assert.Equal(t, offset, part.OffsetMB) assert.Equal(t, size, part.SizeMB) } @@ -132,18 +132,18 @@ func verifyMetadataDevicePartition(t *testing.T, info *MetadataDeviceInfo, index func TestMetadataGetPartitionArgs(t *testing.T) { metadata := NewMetadataDeviceInfo("sda") - e1 := NewPerfSchemeEntry() + e1 := NewPerfSchemeEntry(Bluestore) e1.ID = 1 e1.OsdUUID = uuid.Must(uuid.NewRandom()) - e2 := NewPerfSchemeEntry() + e2 := NewPerfSchemeEntry(Bluestore) e2.ID = 2 e2.OsdUUID = uuid.Must(uuid.NewRandom()) - bluestoreConfig := BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2} - err := PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, bluestoreConfig) + storeConfig := StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2} + err := PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, storeConfig) assert.Nil(t, err) - err = PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, bluestoreConfig) + err = PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, storeConfig) assert.Nil(t, err) expectedArgs := []string{ @@ -160,19 +160,19 @@ func TestMetadataGetPartitionArgs(t *testing.T) { } func TestSchemeEntryGetPartitionArgs(t *testing.T) { - e1 := NewPerfSchemeEntry() + e1 := NewPerfSchemeEntry(Bluestore) e1.ID = 1 e1.OsdUUID = uuid.Must(uuid.NewRandom()) - bluestoreConfig := BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2} - err := PopulateCollocatedPerfSchemeEntry(e1, "sdb", bluestoreConfig) + storeConfig := StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2} + err := PopulateCollocatedPerfSchemeEntry(e1, "sdb", storeConfig) assert.Nil(t, err) expectedArgs := []string{ - "--new=1:2048:+2048", "--change-name=1:ROOK-OSD1-WAL", fmt.Sprintf("--partition-guid=1:%s", e1.Partitions[WalPartitionName].PartitionUUID), - "--new=2:4096:+4096", "--change-name=2:ROOK-OSD1-DB", fmt.Sprintf("--partition-guid=2:%s", e1.Partitions[DatabasePartitionName].PartitionUUID), - "--largest-new=3", "--change-name=3:ROOK-OSD1-BLOCK", fmt.Sprintf("--partition-guid=3:%s", e1.Partitions[BlockPartitionName].PartitionUUID), - fmt.Sprintf("--disk-guid=%s", e1.Partitions[BlockPartitionName].DiskUUID), "/dev/sdb", + "--new=1:2048:+2048", "--change-name=1:ROOK-OSD1-WAL", fmt.Sprintf("--partition-guid=1:%s", e1.Partitions[WalPartitionType].PartitionUUID), + "--new=2:4096:+4096", "--change-name=2:ROOK-OSD1-DB", fmt.Sprintf("--partition-guid=2:%s", e1.Partitions[DatabasePartitionType].PartitionUUID), + "--largest-new=3", "--change-name=3:ROOK-OSD1-BLOCK", fmt.Sprintf("--partition-guid=3:%s", e1.Partitions[BlockPartitionType].PartitionUUID), + fmt.Sprintf("--disk-guid=%s", e1.Partitions[BlockPartitionType].DiskUUID), "/dev/sdb", } // get the partition args and verify against expected diff --git a/pkg/operator/api/api.go b/pkg/operator/api/api.go index e0c747169..366141599 100644 --- a/pkg/operator/api/api.go +++ b/pkg/operator/api/api.go @@ -99,8 +99,7 @@ func (c *Cluster) makeDeployment() *extensions.Deployment { func (c *Cluster) apiContainer() v1.Container { - command := fmt.Sprintf("/usr/bin/rookd api --data-dir=%s --port=%d --container-version=%s", - k8sutil.DataDir, model.Port, c.Version) + command := fmt.Sprintf("/usr/bin/rookd api --config-dir=%s --port=%d --container-version=%s", k8sutil.DataDir, model.Port, c.Version) return v1.Container{ // TODO: fix "sleep 5". // Without waiting some time, there is highly probable flakes in network setup. diff --git a/pkg/operator/api/api_test.go b/pkg/operator/api/api_test.go index 7f4582253..09babd55e 100644 --- a/pkg/operator/api/api_test.go +++ b/pkg/operator/api/api_test.go @@ -78,8 +78,8 @@ func TestPodSpecs(t *testing.T) { for _, v := range cont.Env { assert.True(t, strings.HasPrefix(v.Name, "ROOKD_")) } - expectedCommand := fmt.Sprintf("/usr/bin/rookd api --data-dir=/var/lib/rook --port=%d --container-version=%s", - model.Port, c.Version) + + expectedCommand := fmt.Sprintf("/usr/bin/rookd api --config-dir=/var/lib/rook --port=%d --container-version=%s", model.Port, c.Version) assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2]) } diff --git a/pkg/operator/cluster.go b/pkg/operator/cluster.go index f1e7c242f..b77f6d5e7 100644 --- a/pkg/operator/cluster.go +++ b/pkg/operator/cluster.go @@ -92,12 +92,12 @@ func (t *clusterTPR) findAllClusters() error { } func (t *clusterTPR) startCluster(c *cluster.Cluster) { - if t.devicesInUse && c.Spec.UseAllDevices { + if t.devicesInUse && c.Spec.Storage.AnyUseAllDevices() { logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace) - c.Spec.UseAllDevices = false + c.Spec.Storage.ClearUseAllDevices() } - if c.Spec.UseAllDevices { + if c.Spec.Storage.AnyUseAllDevices() { t.devicesInUse = true } diff --git a/pkg/operator/cluster/cluster.go b/pkg/operator/cluster/cluster.go index 2668fea17..9d46d00e6 100644 --- a/pkg/operator/cluster/cluster.go +++ b/pkg/operator/cluster/cluster.go @@ -92,7 +92,7 @@ func (c *Cluster) CreateInstance() error { } // Start the OSDs - c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices) + c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.Storage, c.Spec.DataDirHostPath) err = c.osds.Start() if err != nil { return fmt.Errorf("failed to start the osds. %+v", err) diff --git a/pkg/operator/cluster/spec.go b/pkg/operator/cluster/spec.go index fa2030d25..21eeefb82 100644 --- a/pkg/operator/cluster/spec.go +++ b/pkg/operator/cluster/spec.go @@ -18,6 +18,8 @@ which also has the apache 2.0 license. */ package cluster +import "github.com/rook/rook/pkg/operator/osd" + type Spec struct { // The namespace where the the rook resources will all be created. Namespace string `json:"namespace"` @@ -30,14 +32,11 @@ type Spec struct { // Paused is to pause the control of the operator for the rook cluster. Paused bool `json:"paused,omitempty"` - // Whether to consume all the storage devices found on a machine - UseAllDevices bool `json:"useAllDevices"` - - // A regular expression to allow more fine-grained selection of devices on nodes across the cluster - DeviceFilter string `json:"deviceFilter"` - // The path on the host where config and data can be persisted. DataDirHostPath string `json:"dataDirHostPath"` + + // A spec for available storage in the cluster and how it should be used + Storage osd.StorageSpec `json:"storage"` } type PoolSpec struct { diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index c4c150c6d..677ab2bc6 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -46,6 +46,10 @@ func RepoPrefixEnvVar() v1.EnvVar { return v1.EnvVar{Name: repoPrefixEnvVar, Value: repoPrefix()} } +func ConfigDirEnvVar() v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_CONFIG_DIR", Value: DataDir} +} + func repoPrefix() string { r := os.Getenv(repoPrefixEnvVar) if r == "" { diff --git a/pkg/cephmgr/osd/partition/log.go b/pkg/operator/k8sutil/volume.go similarity index 55% rename from pkg/cephmgr/osd/partition/log.go rename to pkg/operator/k8sutil/volume.go index 6b486e79a..7193610fb 100644 --- a/pkg/cephmgr/osd/partition/log.go +++ b/pkg/operator/k8sutil/volume.go @@ -13,8 +13,22 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package partition +package k8sutil -import "github.com/coreos/pkg/capnslog" +import ( + "path/filepath" + "strings" +) -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephosdpart") +func PathToVolumeName(path string) string { + // kubernetes volume names must match this regex: [a-z0-9]([-a-z0-9]*[a-z0-9])? + + // first replace all filepath separators with hyphens + volumeName := strings.Replace(path, string(filepath.Separator), "-", -1) + + // trim any leading/trailing hyphens + volumeName = strings.TrimPrefix(volumeName, "-") + volumeName = strings.TrimSuffix(volumeName, "-") + + return volumeName +} diff --git a/pkg/operator/mds/mds.go b/pkg/operator/mds/mds.go index f67483d2e..5347616c4 100644 --- a/pkg/operator/mds/mds.go +++ b/pkg/operator/mds/mds.go @@ -150,7 +150,7 @@ func (c *Cluster) makeDeployment(id string) *extensions.Deployment { func (c *Cluster) mdsContainer(id string) v1.Container { - command := fmt.Sprintf("/usr/bin/rookd mds --data-dir=%s --mds-id=%s ", + command := fmt.Sprintf("/usr/bin/rookd mds --config-dir=%s --mds-id=%s ", k8sutil.DataDir, id) return v1.Container{ // TODO: fix "sleep 5". diff --git a/pkg/operator/mds/mds_test.go b/pkg/operator/mds/mds_test.go index 095d8b0ce..e2ee98ff5 100644 --- a/pkg/operator/mds/mds_test.go +++ b/pkg/operator/mds/mds_test.go @@ -86,7 +86,7 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, 1, len(cont.VolumeMounts)) assert.Equal(t, 5, len(cont.Env)) - expectedCommand := fmt.Sprintf("/usr/bin/rookd mds --data-dir=/var/lib/rook --mds-id=%s ", + expectedCommand := fmt.Sprintf("/usr/bin/rookd mds --config-dir=/var/lib/rook --mds-id=%s ", mdsID) assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2]) diff --git a/pkg/operator/mon/pod.go b/pkg/operator/mon/pod.go index 1f12c9b49..5a6c68509 100644 --- a/pkg/operator/mon/pod.go +++ b/pkg/operator/mon/pod.go @@ -83,7 +83,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod { } func (c *Cluster) monContainer(config *MonConfig, fsid string) v1.Container { - command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --port=%d --fsid=%s", + command := fmt.Sprintf("/usr/bin/rookd mon --config-dir=%s --name=%s --port=%d --fsid=%s", k8sutil.DataDir, config.Name, config.Port, fsid) return v1.Container{ diff --git a/pkg/operator/mon/pod_test.go b/pkg/operator/mon/pod_test.go index 16840a3f7..630b43a8e 100644 --- a/pkg/operator/mon/pod_test.go +++ b/pkg/operator/mon/pod_test.go @@ -62,7 +62,7 @@ func testPodSpec(t *testing.T, dataDir string) { assert.Equal(t, 1, len(cont.VolumeMounts)) assert.Equal(t, 5, len(cont.Env)) - expectedCommand := fmt.Sprintf("/usr/bin/rookd mon --data-dir=/var/lib/rook --name=%s --port=%d --fsid=%s", + expectedCommand := fmt.Sprintf("/usr/bin/rookd mon --config-dir=/var/lib/rook --name=%s --port=%d --fsid=%s", config.Name, config.Port, c.clusterInfo.FSID) assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2]) diff --git a/pkg/operator/osd/osd.go b/pkg/operator/osd/osd.go index 878bf3ace..305cb601d 100644 --- a/pkg/operator/osd/osd.go +++ b/pkg/operator/osd/osd.go @@ -18,16 +18,22 @@ package osd import ( "fmt" + "strings" + + "strconv" + "github.com/rook/rook/pkg/operator/k8sutil" opmon "github.com/rook/rook/pkg/operator/mon" "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/errors" + "k8s.io/client-go/pkg/api/unversioned" "k8s.io/client-go/pkg/api/v1" extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1" ) const ( - appName = "osd" + appName = "osd" + appNameFmt = "osd-%s" ) type Cluster struct { @@ -35,50 +41,112 @@ type Cluster struct { Namespace string Keyring string Version string - useAllDevices bool + Storage StorageSpec dataDirHostPath string - deviceFilter string } -func New(clientset kubernetes.Interface, namespace, version, deviceFilter, dataDirHostPath string, useAllDevices bool) *Cluster { +func New(clientset kubernetes.Interface, namespace, version string, storageSpec StorageSpec, dataDirHostPath string) *Cluster { return &Cluster{ clientset: clientset, Namespace: namespace, Version: version, - deviceFilter: deviceFilter, + Storage: storageSpec, dataDirHostPath: dataDirHostPath, - useAllDevices: useAllDevices, } } func (c *Cluster) Start() error { - logger.Infof("start running osds") + logger.Infof("start running osds in namespace %s", c.Namespace) - ds, err := c.makeDaemonSet() - _, err = c.clientset.Extensions().DaemonSets(c.Namespace).Create(ds) - if err != nil { - if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create osd daemon set. %+v", err) + if c.Storage.UseAllNodes { + // make a daemonset for all nodes in the cluster + ds := c.makeDaemonSet(c.Storage.Selection, c.Storage.Config) + _, err := c.clientset.Extensions().DaemonSets(c.Namespace).Create(ds) + if err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create osd daemon set. %+v", err) + } + logger.Infof("osd daemon set already exists") + } else { + logger.Infof("osd daemon set started") } - logger.Infof("osd daemon set already exists") } else { - logger.Infof("osd daemon set started") + for i := range c.Storage.Nodes { + // fully resolve the storage config for this node + n := c.Storage.resolveNode(c.Storage.Nodes[i].Name) + + // create the replicaSet that will run the OSDs for this node + rs := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config) + _, err := c.clientset.Extensions().ReplicaSets(c.Namespace).Create(rs) + if err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create osd replica set for node %s. %+v", n.Name, err) + } + logger.Infof("osd replica set already exists for node %s", n.Name) + } else { + logger.Infof("osd replica set started for node %s", n.Name) + } + } } return nil } -func (c *Cluster) makeDaemonSet() (*extensions.DaemonSet, error) { +func (c *Cluster) makeDaemonSet(selection Selection, config Config) *extensions.DaemonSet { ds := &extensions.DaemonSet{} ds.Name = appName ds.Namespace = c.Namespace + podSpec := c.podTemplateSpec(nil, nil, selection, config) + + ds.Spec = extensions.DaemonSetSpec{Template: podSpec} + return ds +} + +func (c *Cluster) makeReplicaSet(nodeName string, devices []Device, directories []Directory, + selection Selection, config Config) *extensions.ReplicaSet { + + rs := &extensions.ReplicaSet{} + rs.Name = fmt.Sprintf(appNameFmt, nodeName) + rs.Namespace = c.Namespace + + podSpec := c.podTemplateSpec(devices, directories, selection, config) + podSpec.Spec.NodeSelector = map[string]string{unversioned.LabelHostname: nodeName} + + replicaCount := int32(1) + + rs.Spec = extensions.ReplicaSetSpec{ + Template: podSpec, + Replicas: &replicaCount, + } + + return rs +} + +func (c *Cluster) podTemplateSpec(devices []Device, directories []Directory, selection Selection, config Config) v1.PodTemplateSpec { + // by default, the data/config dir will be an empty volume dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} if c.dataDirHostPath != "" { + // the user has specified a host path to use for the data dir, use that instead dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}} } - podSpec := v1.PodTemplateSpec{ + // create volume config for the data dir and /dev so the pod can access devices on the host + volumes := []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}, + {Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}, + } + + // add each OSD directory as another host path volume source + for _, d := range directories { + dirVolume := v1.Volume{ + Name: k8sutil.PathToVolumeName(d.Path), + VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: d.Path}}, + } + volumes = append(volumes, dirVolume) + } + + return v1.PodTemplateSpec{ ObjectMeta: v1.ObjectMeta{ Name: appName, Labels: map[string]string{ @@ -88,48 +156,124 @@ func (c *Cluster) makeDaemonSet() (*extensions.DaemonSet, error) { Annotations: map[string]string{}, }, Spec: v1.PodSpec{ - Containers: []v1.Container{c.osdContainer()}, + Containers: []v1.Container{c.osdContainer(devices, directories, selection, config)}, RestartPolicy: v1.RestartPolicyAlways, - Volumes: []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}, - {Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}, - }, + Volumes: volumes, }, } - - ds.Spec = extensions.DaemonSetSpec{Template: podSpec} - - return ds, nil } -func (c *Cluster) osdContainer() v1.Container { +func (c *Cluster) osdContainer(devices []Device, directories []Directory, selection Selection, config Config) v1.Container { - command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s ", k8sutil.DataDir) - var devices string - if c.deviceFilter != "" { - devices = c.deviceFilter - } else if c.useAllDevices { - devices = "all" + command := "/usr/bin/rookd osd" + + envVars := []v1.EnvVar{ + opmon.ClusterNameEnvVar(), + opmon.MonEndpointEnvVar(), + opmon.MonSecretEnvVar(), + opmon.AdminSecretEnvVar(), + k8sutil.ConfigDirEnvVar(), + } + + // only 1 of device list, device filter and use all devices can be specified. We prioritize in that order. + if len(devices) > 0 { + deviceNames := make([]string, len(devices)) + for i := range devices { + deviceNames[i] = devices[i].Name + } + envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ","))) + } else if selection.DeviceFilter != "" { + envVars = append(envVars, deviceFilterEnvVar(selection.DeviceFilter)) + } else if selection.getUseAllDevices() { + envVars = append(envVars, deviceFilterEnvVar("all")) + } + + if selection.MetadataDevice != "" { + envVars = append(envVars, metadataDeviceEnvVar(selection.MetadataDevice)) + } + + volumeMounts := []v1.VolumeMount{ + {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, + {Name: "devices", MountPath: "/dev"}, + } + + if len(directories) > 0 { + // for each directory the user has specified, create a volume mount and pass it to the pod via cmd line arg + dirPaths := make([]string, len(directories)) + for i := range directories { + dpath := directories[i].Path + dirPaths[i] = dpath + volumeMounts = append(volumeMounts, v1.VolumeMount{Name: k8sutil.PathToVolumeName(dpath), MountPath: dpath}) + } + + envVars = append(envVars, dataDirectoriesEnvVar(strings.Join(dirPaths, ","))) + } + + if config.StoreConfig.StoreType != "" { + envVars = append(envVars, osdStoreEnvVar(config.StoreConfig.StoreType)) + } + + if config.StoreConfig.DatabaseSizeMB != 0 { + envVars = append(envVars, osdDatabaseSizeEnvVar(config.StoreConfig.DatabaseSizeMB)) + } + + if config.StoreConfig.WalSizeMB != 0 { + envVars = append(envVars, osdWalSizeEnvVar(config.StoreConfig.WalSizeMB)) + } + + if config.StoreConfig.JournalSizeMB != 0 { + envVars = append(envVars, osdJournalSizeEnvVar(config.StoreConfig.JournalSizeMB)) + } + + if config.Location != "" { + envVars = append(envVars, locationEnvVar(config.Location)) } privileged := true return v1.Container{ // TODO: fix "sleep 5". // Without waiting some time, there is highly probable flakes in network setup. - Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)}, - Name: appName, - Image: k8sutil.MakeRookImage(c.Version), - VolumeMounts: []v1.VolumeMount{ - {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, - {Name: "devices", MountPath: "/dev"}, - }, - Env: []v1.EnvVar{ - v1.EnvVar{Name: "ROOKD_DATA_DEVICES", Value: devices}, - opmon.ClusterNameEnvVar(), - opmon.MonEndpointEnvVar(), - opmon.MonSecretEnvVar(), - opmon.AdminSecretEnvVar(), - }, + Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)}, + Name: appName, + Image: k8sutil.MakeRookImage(c.Version), + VolumeMounts: volumeMounts, + Env: envVars, SecurityContext: &v1.SecurityContext{Privileged: &privileged}, } } + +func dataDevicesEnvVar(dataDevices string) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_DATA_DEVICES", Value: dataDevices} +} + +func deviceFilterEnvVar(filter string) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_DATA_DEVICE_FILTER", Value: filter} +} + +func metadataDeviceEnvVar(metadataDevice string) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_METADATA_DEVICE", Value: metadataDevice} +} + +func dataDirectoriesEnvVar(dataDirectories string) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_DATA_DIRECTORIES", Value: dataDirectories} +} + +func osdStoreEnvVar(osdStore string) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_OSD_STORE", Value: osdStore} +} + +func osdDatabaseSizeEnvVar(databaseSize int) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_OSD_DATABASE_SIZE", Value: strconv.Itoa(databaseSize)} +} + +func osdWalSizeEnvVar(walSize int) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_OSD_WAL_SIZE", Value: strconv.Itoa(walSize)} +} + +func osdJournalSizeEnvVar(journalSize int) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_OSD_JOURNAL_SIZE", Value: strconv.Itoa(journalSize)} +} + +func locationEnvVar(location string) v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_LOCATION", Value: location} +} diff --git a/pkg/operator/osd/osd_test.go b/pkg/operator/osd/osd_test.go index 5270a66fd..1cd8924f2 100644 --- a/pkg/operator/osd/osd_test.go +++ b/pkg/operator/osd/osd_test.go @@ -16,6 +16,7 @@ limitations under the License. package osd import ( + "strconv" "strings" "testing" @@ -23,11 +24,14 @@ import ( "k8s.io/client-go/pkg/api/v1" "github.com/stretchr/testify/assert" + "k8s.io/client-go/pkg/api/unversioned" + + cephosd "github.com/rook/rook/pkg/cephmgr/osd" ) func TestStartDaemonset(t *testing.T) { clientset := fake.NewSimpleClientset() - c := New(clientset, "ns", "myversion", "", "", false) + c := New(clientset, "ns", "myversion", StorageSpec{}, "") // Start the first time err := c.Start() @@ -46,44 +50,140 @@ func TestDaemonset(t *testing.T) { } func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool) { - clientset := fake.NewSimpleClientset() - c := New(clientset, "ns", "myversion", deviceFilter, dataDir, allDevices) - - daemonSet, err := c.makeDaemonSet() - assert.Nil(t, err) - assert.NotNil(t, daemonSet) - assert.Equal(t, "osd", daemonSet.Name) - assert.Equal(t, c.Namespace, daemonSet.Namespace) - assert.Equal(t, v1.RestartPolicyAlways, daemonSet.Spec.Template.Spec.RestartPolicy) - assert.Equal(t, 2, len(daemonSet.Spec.Template.Spec.Volumes)) - assert.Equal(t, "rook-data", daemonSet.Spec.Template.Spec.Volumes[0].Name) - assert.Equal(t, "devices", daemonSet.Spec.Template.Spec.Volumes[1].Name) - if dataDir == "" { - assert.NotNil(t, daemonSet.Spec.Template.Spec.Volumes[0].EmptyDir) - assert.Nil(t, daemonSet.Spec.Template.Spec.Volumes[0].HostPath) - } else { - assert.Nil(t, daemonSet.Spec.Template.Spec.Volumes[0].EmptyDir) - assert.Equal(t, dataDir, daemonSet.Spec.Template.Spec.Volumes[0].HostPath.Path) + storageSpec := StorageSpec{ + Selection: Selection{UseAllDevices: &allDevices, DeviceFilter: deviceFilter}, + Nodes: []Node{{Name: "node1"}}, } - assert.Equal(t, "osd", daemonSet.Spec.Template.ObjectMeta.Name) - assert.Equal(t, "osd", daemonSet.Spec.Template.ObjectMeta.Labels["app"]) - assert.Equal(t, c.Namespace, daemonSet.Spec.Template.ObjectMeta.Labels["rook_cluster"]) - assert.Equal(t, 0, len(daemonSet.Spec.Template.ObjectMeta.Annotations)) + clientset := fake.NewSimpleClientset() + c := New(clientset, "ns", "myversion", storageSpec, dataDir) - cont := daemonSet.Spec.Template.Spec.Containers[0] + n := c.Storage.resolveNode(storageSpec.Nodes[0].Name) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config) + assert.NotNil(t, replicaSet) + assert.Equal(t, "osd-node1", replicaSet.Name) + assert.Equal(t, c.Namespace, replicaSet.Namespace) + assert.Equal(t, int32(1), *(replicaSet.Spec.Replicas)) + assert.Equal(t, "node1", replicaSet.Spec.Template.Spec.NodeSelector[unversioned.LabelHostname]) + assert.Equal(t, v1.RestartPolicyAlways, replicaSet.Spec.Template.Spec.RestartPolicy) + assert.Equal(t, 2, len(replicaSet.Spec.Template.Spec.Volumes)) + assert.Equal(t, "rook-data", replicaSet.Spec.Template.Spec.Volumes[0].Name) + assert.Equal(t, "devices", replicaSet.Spec.Template.Spec.Volumes[1].Name) + if dataDir == "" { + assert.NotNil(t, replicaSet.Spec.Template.Spec.Volumes[0].EmptyDir) + assert.Nil(t, replicaSet.Spec.Template.Spec.Volumes[0].HostPath) + } else { + assert.Nil(t, replicaSet.Spec.Template.Spec.Volumes[0].EmptyDir) + assert.Equal(t, dataDir, replicaSet.Spec.Template.Spec.Volumes[0].HostPath.Path) + } + + assert.Equal(t, "osd", replicaSet.Spec.Template.ObjectMeta.Name) + assert.Equal(t, "osd", replicaSet.Spec.Template.ObjectMeta.Labels["app"]) + assert.Equal(t, c.Namespace, replicaSet.Spec.Template.ObjectMeta.Labels["rook_cluster"]) + assert.Equal(t, 0, len(replicaSet.Spec.Template.ObjectMeta.Annotations)) + + cont := replicaSet.Spec.Template.Spec.Containers[0] assert.Equal(t, "quay.io/rook/rookd:myversion", cont.Image) assert.Equal(t, 2, len(cont.VolumeMounts)) - assert.Equal(t, 5, len(cont.Env)) - expectedCommand := "/usr/bin/rookd osd --data-dir=/var/lib/rook " + expectedCommand := "/usr/bin/rookd osd" assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2]) - devices := cont.Env[0].Value + + // verify the config dir env var + verifyEnvVar(t, cont.Env, "ROOKD_CONFIG_DIR", "/var/lib/rook", true) + + // verify the osd store type env var uses the default + verifyEnvVar(t, cont.Env, "ROOKD_OSD_STORE", cephosd.DefaultStore, true) + + // verify the device filter env var if deviceFilter != "" { - assert.Equal(t, deviceFilter, devices) + verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", deviceFilter, true) } else if allDevices { - assert.Equal(t, "all", devices) + verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", "all", true) } else { - assert.Equal(t, "", devices) + verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", "", false) } } + +func verifyEnvVar(t *testing.T, envVars []v1.EnvVar, expectedName, expectedValue string, expectedFound bool) { + found := false + for _, envVar := range envVars { + if envVar.Name == expectedName { + assert.Equal(t, expectedValue, envVar.Value) + found = true + break + } + } + + assert.Equal(t, expectedFound, found) +} + +func TestStorageSpecDevicesAndDirectories(t *testing.T) { + storageSpec := StorageSpec{ + Config: Config{}, + Nodes: []Node{ + { + Name: "node1", + Devices: []Device{{Name: "sda"}}, + Directories: []Directory{{Path: "/rook/dir1"}}, + }, + }, + } + + clientset := fake.NewSimpleClientset() + c := New(clientset, "ns", "myversion", storageSpec, "") + + n := c.Storage.resolveNode(storageSpec.Nodes[0].Name) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config) + assert.NotNil(t, replicaSet) + + // pod spec should have a volume for the given dir + podSpec := replicaSet.Spec.Template.Spec + assert.Equal(t, 3, len(podSpec.Volumes)) + assert.Equal(t, "rook-dir1", podSpec.Volumes[2].Name) + assert.Equal(t, "/rook/dir1", podSpec.Volumes[2].VolumeSource.HostPath.Path) + + // container should have a volume mount for the given dir + container := podSpec.Containers[0] + assert.Equal(t, "rook-dir1", container.VolumeMounts[2].Name) + assert.Equal(t, "/rook/dir1", container.VolumeMounts[2].MountPath) + + // container command should have the given dir and device + verifyEnvVar(t, container.Env, "ROOKD_DATA_DIRECTORIES", "/rook/dir1", true) + verifyEnvVar(t, container.Env, "ROOKD_DATA_DEVICES", "sda", true) +} + +func TestStorageSpecConfig(t *testing.T) { + storageSpec := StorageSpec{ + Config: Config{}, + Nodes: []Node{ + { + Name: "node1", + Config: Config{ + Location: "rack=foo", + StoreConfig: cephosd.StoreConfig{ + StoreType: cephosd.Bluestore, + DatabaseSizeMB: 10, + WalSizeMB: 20, + JournalSizeMB: 30, + }, + }, + }, + }, + } + + clientset := fake.NewSimpleClientset() + c := New(clientset, "ns", "myversion", storageSpec, "") + + n := c.Storage.resolveNode(storageSpec.Nodes[0].Name) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config) + assert.NotNil(t, replicaSet) + + container := replicaSet.Spec.Template.Spec.Containers[0] + assert.NotNil(t, container) + verifyEnvVar(t, container.Env, "ROOKD_OSD_STORE", cephosd.Bluestore, true) + verifyEnvVar(t, container.Env, "ROOKD_OSD_DATABASE_SIZE", strconv.Itoa(10), true) + verifyEnvVar(t, container.Env, "ROOKD_OSD_WAL_SIZE", strconv.Itoa(20), true) + verifyEnvVar(t, container.Env, "ROOKD_OSD_JOURNAL_SIZE", strconv.Itoa(30), true) + verifyEnvVar(t, container.Env, "ROOKD_LOCATION", "rack=foo", true) +} diff --git a/pkg/operator/osd/spec.go b/pkg/operator/osd/spec.go new file mode 100644 index 000000000..bc6e82483 --- /dev/null +++ b/pkg/operator/osd/spec.go @@ -0,0 +1,58 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package osd + +import ( + cephosd "github.com/rook/rook/pkg/cephmgr/osd" +) + +type StorageSpec struct { + Nodes []Node `json:"nodes,omitempty"` + UseAllNodes bool `json:"useAllNodes,omitempty"` + Selection + Config +} + +type Node struct { + Name string `json:"name,omitempty"` + Devices []Device `json:"devices,omitempty"` + Directories []Directory `json:"directories,omitempty"` + Selection + Config +} + +type Device struct { + Name string `json:"name,omitempty"` +} + +type Directory struct { + Path string `json:"path,omitempty"` +} + +type Selection struct { + // Whether to consume all the storage devices found on a machine + UseAllDevices *bool `json:"useAllDevices,omitempty"` + + // A regular expression to allow more fine-grained selection of devices on nodes across the cluster + DeviceFilter string `json:"deviceFilter,omitempty"` + + MetadataDevice string `json:"metadataDevice,omitempty"` +} + +type Config struct { + StoreConfig cephosd.StoreConfig `json:"storeConfig,omitempty"` + Location string `json:"location,omitempty"` +} diff --git a/pkg/operator/osd/spec_test.go b/pkg/operator/osd/spec_test.go new file mode 100644 index 000000000..3d2c9b116 --- /dev/null +++ b/pkg/operator/osd/spec_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package osd + +import ( + "encoding/json" + "testing" + + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + + cephosd "github.com/rook/rook/pkg/cephmgr/osd" +) + +func TestStorageSpecMarshal(t *testing.T) { + specYaml := []byte(`useAllNodes: true +useAllDevices: false +deviceFilter: "^nvme." +metadataDevice: +location: "region=us-west,datacenter=delmar" +storeConfig: + storeType: bluestore + journalSizeMB: 1024 + databaseSizeMB: 1024 +nodes: +- name: "node1" + storeConfig: + storeType: filestore + directories: + - path: "/rook/dir1" +- name: "node2" + deviceFilter: "^sd."`) + + // convert the raw spec yaml into JSON + rawJson, err := yaml.YAMLToJSON(specYaml) + assert.Nil(t, err) + + // unmarshal the JSON into a strongly typed storage spec object + var storageSpec StorageSpec + err = json.Unmarshal(rawJson, &storageSpec) + assert.Nil(t, err) + + // the unmarshalled storage spec should equal the expected spec below + expectedSpec := StorageSpec{ + UseAllNodes: true, + Selection: Selection{ + UseAllDevices: newBool(false), + DeviceFilter: "^nvme.", + }, + Config: Config{ + Location: "region=us-west,datacenter=delmar", + StoreConfig: cephosd.StoreConfig{ + DatabaseSizeMB: 1024, + JournalSizeMB: 1024, + StoreType: cephosd.Bluestore, + }, + }, + Nodes: []Node{ + Node{ + Name: "node1", + Directories: []Directory{{Path: "/rook/dir1"}}, + Config: Config{ + StoreConfig: cephosd.StoreConfig{ + StoreType: cephosd.Filestore, + }, + }, + }, + Node{ + Name: "node2", + Selection: Selection{ + DeviceFilter: "^sd.", + }, + }, + }, + } + + assert.Equal(t, expectedSpec, storageSpec) +} diff --git a/pkg/operator/osd/storage.go b/pkg/operator/osd/storage.go new file mode 100644 index 000000000..af392a952 --- /dev/null +++ b/pkg/operator/osd/storage.go @@ -0,0 +1,121 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package osd + +import ( + cephosd "github.com/rook/rook/pkg/cephmgr/osd" +) + +func (s *StorageSpec) AnyUseAllDevices() bool { + if s.Selection.getUseAllDevices() { + return true + } + + for _, n := range s.Nodes { + if n.Selection.getUseAllDevices() { + return true + } + } + + return false +} + +func (s *StorageSpec) ClearUseAllDevices() { + clear := false + s.Selection.UseAllDevices = &clear + for i := range s.Nodes { + s.Nodes[i].Selection.UseAllDevices = &clear + } +} + +// Fully resolves the config of the given node name, taking into account cluster level and node level specified config. +// In general, the more fine grained the configuration is specified, the more precedence it takes. Fully resolved +// configuration for the node has the following order of precedence. +// 1) Node (config defined on the node itself) +// 2) Cluster (config defined on the cluster) +// 3) Default values (if no config exists for the node or cluster) +func (s *StorageSpec) resolveNode(nodeName string) *Node { + // find the requested storage node first, if it exists + var node *Node + for i := range s.Nodes { + if s.Nodes[i].Name == nodeName { + node = &(s.Nodes[i]) + break + } + } + + if node == nil { + // a node with the given name was not found + return nil + } + + // now resolve all properties that haven't already been set on the node + s.resolveNodeSelection(node) + s.resolveNodeConfig(node) + + return node +} + +func (s *StorageSpec) resolveNodeSelection(node *Node) { + resolveString(&(node.Selection.DeviceFilter), s.Selection.DeviceFilter, "") + resolveString(&(node.Selection.MetadataDevice), s.Selection.MetadataDevice, "") + + if node.Selection.UseAllDevices == nil { + if s.Selection.UseAllDevices != nil { + // the node does not have a value specified for use all devices, but the cluster does. Use the cluster's. + node.Selection.UseAllDevices = s.Selection.UseAllDevices + } else { + // neither node nor cluster have a value set for use all devices, use the default value. + node.Selection.UseAllDevices = newBool(false) + } + } +} + +func (s *StorageSpec) resolveNodeConfig(node *Node) { + resolveString(&(node.Config.StoreConfig.StoreType), s.Config.StoreConfig.StoreType, cephosd.DefaultStore) + resolveInt(&(node.Config.StoreConfig.DatabaseSizeMB), s.Config.StoreConfig.DatabaseSizeMB, 0) + resolveInt(&(node.Config.StoreConfig.WalSizeMB), s.Config.StoreConfig.WalSizeMB, 0) + resolveInt(&(node.Config.StoreConfig.JournalSizeMB), s.Config.StoreConfig.JournalSizeMB, 0) + resolveString(&(node.Config.Location), s.Config.Location, "") +} + +func (s *Selection) getUseAllDevices() bool { + return s.UseAllDevices != nil && *(s.UseAllDevices) +} + +func resolveString(setting *string, parent, defaultVal string) { + if *setting == "" { + if parent != "" { + *setting = parent + } else { + *setting = defaultVal + } + } +} + +func resolveInt(setting *int, parent, defaultVal int) { + if *setting == 0 { + if parent != 0 { + *setting = parent + } else { + *setting = defaultVal + } + } +} + +func newBool(val bool) *bool { + return &val +} diff --git a/pkg/operator/osd/storage_test.go b/pkg/operator/osd/storage_test.go new file mode 100644 index 000000000..6be7c2f8f --- /dev/null +++ b/pkg/operator/osd/storage_test.go @@ -0,0 +1,169 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package osd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + cephosd "github.com/rook/rook/pkg/cephmgr/osd" +) + +func TestResolveNodeNotExist(t *testing.T) { + + // a non existing node should return nil + storageSpec := StorageSpec{} + node := storageSpec.resolveNode("fake node") + assert.Nil(t, node) + + // a node with no properties defined should inherit them from the cluster storage spec + storageSpec = StorageSpec{ + Selection: Selection{ + DeviceFilter: "^sd.", + MetadataDevice: "nvme01", + }, + Config: Config{ + Location: "root=default,row=a,rack=a2,chassis=a2a,host=a2a1", + StoreConfig: cephosd.StoreConfig{ + StoreType: cephosd.Bluestore, + DatabaseSizeMB: 1024, + WalSizeMB: 128, + JournalSizeMB: 2048, + }, + }, + Nodes: []Node{ + {Name: "node1"}, // node gets nothing but its name set + }, + } + node = storageSpec.resolveNode("node1") + assert.NotNil(t, node) + assert.Equal(t, "^sd.", node.Selection.DeviceFilter) + assert.Equal(t, "nvme01", node.Selection.MetadataDevice) + assert.False(t, node.Selection.getUseAllDevices()) + assert.Equal(t, "root=default,row=a,rack=a2,chassis=a2a,host=a2a1", node.Config.Location) + assert.Equal(t, cephosd.Bluestore, node.Config.StoreConfig.StoreType) + assert.Equal(t, 1024, node.Config.StoreConfig.DatabaseSizeMB) + assert.Equal(t, 128, node.Config.StoreConfig.WalSizeMB) + assert.Equal(t, 2048, node.Config.StoreConfig.JournalSizeMB) +} + +func TestResolveNodeInherentFromCluster(t *testing.T) { + // a node with no properties defined should inherit them from the cluster storage spec + storageSpec := StorageSpec{ + Selection: Selection{ + DeviceFilter: "^sd.", + MetadataDevice: "nvme01", + }, + Config: Config{ + Location: "root=default,row=a,rack=a2,chassis=a2a,host=a2a1", + StoreConfig: cephosd.StoreConfig{ + StoreType: cephosd.Bluestore, + DatabaseSizeMB: 1024, + WalSizeMB: 128, + JournalSizeMB: 2048, + }, + }, + Nodes: []Node{ + {Name: "node1"}, // node gets nothing but its name set + }, + } + node := storageSpec.resolveNode("node1") + assert.NotNil(t, node) + assert.Equal(t, "^sd.", node.Selection.DeviceFilter) + assert.Equal(t, "nvme01", node.Selection.MetadataDevice) + assert.False(t, node.Selection.getUseAllDevices()) + assert.Equal(t, "root=default,row=a,rack=a2,chassis=a2a,host=a2a1", node.Config.Location) + assert.Equal(t, cephosd.Bluestore, node.Config.StoreConfig.StoreType) + assert.Equal(t, 1024, node.Config.StoreConfig.DatabaseSizeMB) + assert.Equal(t, 128, node.Config.StoreConfig.WalSizeMB) + assert.Equal(t, 2048, node.Config.StoreConfig.JournalSizeMB) +} + +func TestResolveNodeDefaultValues(t *testing.T) { + + // a node with no properties and none defined in the cluster storage spec should get the default values + storageSpec := StorageSpec{ + Nodes: []Node{ + {Name: "node1"}, // node gets nothing but its name set + }, + } + + node := storageSpec.resolveNode("node1") + assert.NotNil(t, node) + assert.Equal(t, "", node.Selection.DeviceFilter) + assert.Equal(t, "", node.Selection.MetadataDevice) + assert.False(t, node.Selection.getUseAllDevices()) + assert.Equal(t, "", node.Config.Location) + assert.Equal(t, cephosd.Filestore, node.Config.StoreConfig.StoreType) + assert.Equal(t, 0, node.Config.StoreConfig.DatabaseSizeMB) + assert.Equal(t, 0, node.Config.StoreConfig.WalSizeMB) + assert.Equal(t, 0, node.Config.StoreConfig.JournalSizeMB) +} + +func TestResolveNodeUseAllDevices(t *testing.T) { + + storageSpec := StorageSpec{ + Selection: Selection{UseAllDevices: newBool(true)}, // UseAllDevices is set to true on the storage spec + Nodes: []Node{ + {Name: "node1"}, // node gets nothing but its name set + }, + } + + node := storageSpec.resolveNode("node1") + assert.NotNil(t, node) + assert.True(t, node.Selection.getUseAllDevices()) +} + +func TestUseAllDevices(t *testing.T) { + storageSpec := StorageSpec{} + assert.False(t, storageSpec.AnyUseAllDevices()) + + storageSpec = StorageSpec{ + Selection: Selection{ + UseAllDevices: newBool(true)}, // UseAllDevices is set to true on the storage spec + } + assert.True(t, storageSpec.AnyUseAllDevices()) + + storageSpec = StorageSpec{ + Selection: Selection{UseAllDevices: newBool(false)}, + Nodes: []Node{ + { + Name: "node1", + Selection: Selection{UseAllDevices: newBool(true)}, + }, + }, + } + assert.True(t, storageSpec.AnyUseAllDevices()) +} + +func TestClearUseAllDevices(t *testing.T) { + // create a storage spec with use all devices set to true for the cluster and for all nodes + storageSpec := StorageSpec{ + Selection: Selection{UseAllDevices: newBool(true)}, + Nodes: []Node{ + { + Name: "node1", + Selection: Selection{UseAllDevices: newBool(true)}, + }, + }, + } + assert.True(t, storageSpec.AnyUseAllDevices()) + + // now clear the use all devices field, it should be cleared from the entire cluster and its nodes + storageSpec.ClearUseAllDevices() + assert.False(t, storageSpec.AnyUseAllDevices()) +} diff --git a/pkg/operator/rgw/rgw.go b/pkg/operator/rgw/rgw.go index d9b708d86..df7f14040 100644 --- a/pkg/operator/rgw/rgw.go +++ b/pkg/operator/rgw/rgw.go @@ -158,7 +158,7 @@ func (c *Cluster) makeDeployment() *extensions.Deployment { func (c *Cluster) rgwContainer() v1.Container { - command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --rgw-port=%d --rgw-host=%s", + command := fmt.Sprintf("/usr/bin/rookd rgw --config-dir=%s --rgw-port=%d --rgw-host=%s", k8sutil.DataDir, cephrgw.RGWPort, cephrgw.DNSName) return v1.Container{ // TODO: fix "sleep 5". diff --git a/pkg/operator/rgw/rgw_test.go b/pkg/operator/rgw/rgw_test.go index 31e9ee465..98dbcd98b 100644 --- a/pkg/operator/rgw/rgw_test.go +++ b/pkg/operator/rgw/rgw_test.go @@ -95,7 +95,7 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, 1, len(cont.VolumeMounts)) assert.Equal(t, 5, len(cont.Env)) - expectedCommand := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=/var/lib/rook --rgw-port=%d --rgw-host=%s", + expectedCommand := fmt.Sprintf("/usr/bin/rookd rgw --config-dir=/var/lib/rook --rgw-port=%d --rgw-host=%s", cephrgw.RGWPort, cephrgw.DNSName) assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2]) diff --git a/pkg/util/exec/exec.go b/pkg/util/exec/exec.go index 74e1d50bc..f9d879fb8 100644 --- a/pkg/util/exec/exec.go +++ b/pkg/util/exec/exec.go @@ -18,6 +18,7 @@ package exec import ( "bufio" "io" + "os" "os/exec" "strings" @@ -30,6 +31,7 @@ type Executor interface { ExecuteCommandPipeline(actionName string, command string) (string, error) ExecuteCommandWithOutput(actionName string, command string, arg ...string) (string, error) ExecuteCommandWithCombinedOutput(actionName string, command string, arg ...string) (string, error) + ExecuteStat(name string) (os.FileInfo, error) } type CommandExecutor struct { @@ -93,6 +95,10 @@ func startCommand(command string, arg ...string) (*exec.Cmd, io.ReadCloser, io.R return cmd, stdout, stderr, err } +func (*CommandExecutor) ExecuteStat(name string) (os.FileInfo, error) { + return os.Stat(name) +} + func logOutput(name string, stdout, stderr io.ReadCloser) { if stdout == nil || stderr == nil { logger.Warningf("failed to collect stdout and stderr") diff --git a/pkg/util/exec/test/mockexec.go b/pkg/util/exec/test/mockexec.go index 591963e82..9301009fe 100644 --- a/pkg/util/exec/test/mockexec.go +++ b/pkg/util/exec/test/mockexec.go @@ -15,7 +15,10 @@ limitations under the License. */ package test -import "os/exec" +import ( + "os" + "os/exec" +) // ******************** MockExecutor ******************** type MockExecutor struct { @@ -24,6 +27,7 @@ type MockExecutor struct { MockExecuteCommandPipeline func(actionName string, command string) (string, error) MockExecuteCommandWithOutput func(actionName string, command string, arg ...string) (string, error) MockExecuteCommandWithCombinedOutput func(actionName string, command string, arg ...string) (string, error) + MockExecuteStat func(name string) (os.FileInfo, error) } func (e *MockExecutor) ExecuteCommand(actionName string, command string, arg ...string) error { @@ -66,3 +70,11 @@ func (e *MockExecutor) ExecuteCommandWithCombinedOutput(actionName string, comma return "", nil } + +func (e *MockExecutor) ExecuteStat(name string) (os.FileInfo, error) { + if e.MockExecuteStat != nil { + return e.MockExecuteStat(name) + } + + return nil, nil +} diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go index 1c2df2a50..1b036613f 100644 --- a/pkg/util/sys/device.go +++ b/pkg/util/sys/device.go @@ -146,7 +146,7 @@ func CreatePartitions(device string, args []string, executor exec.Executor) erro func FormatDevice(devicePath string, executor exec.Executor) error { cmd := fmt.Sprintf("mkfs.ext4 %s", devicePath) - if err := executor.ExecuteCommand(cmd, "sudo", "mkfs.ext4", devicePath); err != nil { + if err := executor.ExecuteCommand(cmd, "mkfs.ext4", devicePath); err != nil { return fmt.Errorf("command %s failed: %+v", cmd, err) } @@ -197,7 +197,7 @@ func MountDevice(devicePath, mountPath string, executor exec.Executor) error { // comma-separated list of mount options passed directly to mount command func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, executor exec.Executor) error { - args := []string{"mount"} + args := []string{} if fstype != "" { args = append(args, "-t", fstype) @@ -212,7 +212,7 @@ func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, execu os.MkdirAll(mountPath, 0755) cmd := fmt.Sprintf("mount %s", devicePath) - if err := executor.ExecuteCommand(cmd, "sudo", args...); err != nil { + if err := executor.ExecuteCommand(cmd, "mount", args...); err != nil { return fmt.Errorf("command %s failed: %+v", cmd, err) } @@ -221,7 +221,7 @@ func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, execu func UnmountDevice(devicePath string, executor exec.Executor) error { cmd := fmt.Sprintf("umount %s", devicePath) - if err := executor.ExecuteCommand(cmd, "sudo", "umount", devicePath); err != nil { + if err := executor.ExecuteCommand(cmd, "umount", devicePath); err != nil { cmdErr, ok := err.(*exec.CommandError) if ok && cmdErr.ExitStatus() == 32 { logger.Infof("ignoring exit status 32 from unmount of device %s, err:%+v", devicePath, cmdErr) diff --git a/pkg/util/sys/device_test.go b/pkg/util/sys/device_test.go index f9a231057..5696e4182 100644 --- a/pkg/util/sys/device_test.go +++ b/pkg/util/sys/device_test.go @@ -94,13 +94,13 @@ func TestMountDeviceWithOptions(t *testing.T) { MockExecuteCommand: func(actionName string, command string, arg ...string) error { switch testCount { case 0: - assert.Equal(t, []string{"mount", "/dev/abc1", "/tmp/mount1"}, arg) + assert.Equal(t, []string{"/dev/abc1", "/tmp/mount1"}, arg) case 1: - assert.Equal(t, []string{"mount", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg) + assert.Equal(t, []string{"-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg) case 2: - assert.Equal(t, []string{"mount", "-t", "myfstype", "/dev/abc1", "/tmp/mount1"}, arg) + assert.Equal(t, []string{"-t", "myfstype", "/dev/abc1", "/tmp/mount1"}, arg) case 3: - assert.Equal(t, []string{"mount", "-t", "myfstype", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg) + assert.Equal(t, []string{"-t", "myfstype", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg) } testCount++