Merge pull request #529 from jbw976/storage-config

Kubernetes operator and cephmgr support for specifying storage resour…
This commit is contained in:
Jared Watts authored and GitHub committed 2017-04-04 15:51:59 -07:00
commit c2d6f34fa8
45 files changed
+1558 -476

No files matched your search

+1 -1
View File
@@ -17,7 +17,7 @@ Rook can be deployed as a standalone service directly on any modern Linux host b
2. Start a one node Rook cluster
```bash
./rookd --data-dir /tmp/rook-test
./rookd --config-dir /tmp/rook-test
```
3. In another console (but in the same path), verify the Rook cluster is ready
+1 -1
View File
@@ -15,7 +15,7 @@ build() {
trap "rm -fr $tmpdir" EXIT
cat <<EOF > $tmpdir/Dockerfile
FROM alpine:3.5
RUN apk add --no-cache gptfdisk util-linux coreutils
RUN apk add --no-cache gptfdisk util-linux coreutils e2fsprogs
COPY root /
ENTRYPOINT ["/usr/bin/rookd"]
EOF
+2 -1
View File
@@ -38,7 +38,8 @@ func TestMountFilesystem(t *testing.T) {
}
e := &exectest.MockExecutor{
MockExecuteCommand: func(actionName string, command string, arg ...string) error {
expectedArgs := []string{"mount", "-t", "ceph", "-o", "name=admin,secret=AQBsCv1X5oD9GhAARHVU9N+kFRWDjyLA1dqzIg==",
assert.Equal(t, "mount", command)
expectedArgs := []string{"-t", "ceph", "-o", "name=admin,secret=AQBsCv1X5oD9GhAARHVU9N+kFRWDjyLA1dqzIg==",
"10.37.129.214:6790:/", "/tmp/myfs1mount"}
assert.Equal(t, expectedArgs, arg)
return nil
+2 -1
View File
@@ -27,7 +27,8 @@ import (
func TestUnmountFilesystem(t *testing.T) {
e := &exectest.MockExecutor{
MockExecuteCommand: func(actionName string, command string, arg ...string) error {
expectedArgs := []string{"umount", "/tmp/myfs1mount"}
assert.Equal(t, "umount", command)
expectedArgs := []string{"/tmp/myfs1mount"}
assert.Equal(t, expectedArgs, arg)
return nil
},
+1 -1
View File
@@ -50,7 +50,7 @@ func init() {
}
func startAPI(cmd *cobra.Command, args []string) error {
if err := flags.VerifyRequiredFlags(apiCmd, []string{"repo-prefix", "namespace", "data-dir", "cluster-name", "mon-endpoints", "mon-secret", "admin-secret"}); err != nil {
if err := flags.VerifyRequiredFlags(apiCmd, []string{"repo-prefix", "namespace", "config-dir", "cluster-name", "mon-endpoints", "mon-secret", "admin-secret"}); err != nil {
return err
}
if apiPort == 0 {
+5 -3
View File
@@ -33,7 +33,7 @@ import (
"github.com/rook/rook/pkg/cephmgr"
"github.com/rook/rook/pkg/cephmgr/cephd"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/cephmgr/osd"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/pkg/util"
@@ -62,6 +62,7 @@ type config struct {
discoveryURL string
etcdMembers string
devices string
directories string
metadataDevice string
dataDir string
adminSecret string
@@ -69,7 +70,7 @@ type config struct {
location string
logLevel capnslog.LogLevel
cephConfigOverride string
bluestoreConfig partition.BluestoreConfig
storeConfig osd.StoreConfig
networkInfo clusterd.NetworkInfo
monEndpoints string
namespace string
@@ -160,7 +161,8 @@ func joinCluster() error {
}
services := []*clusterd.ClusterService{
cephmgr.NewCephService(cephd.New(), cfg.devices, cfg.metadataDevice, cfg.forceFormat, cfg.location, cfg.adminSecret, cfg.bluestoreConfig),
cephmgr.NewCephService(cephd.New(), cfg.devices, cfg.metadataDevice, cfg.directories,
cfg.forceFormat, cfg.location, cfg.adminSecret, cfg.storeConfig),
}
cfg.nodeID, err = util.LoadPersistedNodeID(cfg.dataDir)
+2 -2
View File
@@ -46,7 +46,7 @@ func addCephFlags(command *cobra.Command) {
command.Flags().StringVar(&clusterInfo.MonitorSecret, "mon-secret", "", "the cephx keyring for monitors")
command.Flags().StringVar(&clusterInfo.AdminSecret, "admin-secret", "", "secret for the admin user (random if not specified)")
command.Flags().StringVar(&cfg.monEndpoints, "mon-endpoints", "", "ceph mon endpoints")
command.Flags().StringVar(&cfg.dataDir, "data-dir", "/var/lib/rook", "directory for storing configuration")
command.Flags().StringVar(&cfg.dataDir, "config-dir", "/var/lib/rook", "directory for storing configuration")
command.Flags().StringVar(&cfg.cephConfigOverride, "ceph-config-override", "", "optional path to a ceph config file that will be appended to the config files that rook generates")
}
@@ -61,7 +61,7 @@ func init() {
}
func startMon(cmd *cobra.Command, args []string) error {
if err := flags.VerifyRequiredFlags(monCmd, []string{"name", "fsid", "mon-secret", "admin-secret", "data-dir", "cluster-name", "private-ipv4"}); err != nil {
if err := flags.VerifyRequiredFlags(monCmd, []string{"name", "fsid", "mon-secret", "admin-secret", "config-dir", "cluster-name", "private-ipv4"}); err != nil {
return err
}
+29 -10
View File
@@ -18,10 +18,11 @@ limitations under the License.
package main
import (
"fmt"
"github.com/rook/rook/pkg/cephmgr/cephd"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/cephmgr/osd"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
@@ -33,18 +34,24 @@ var osdCmd = &cobra.Command{
Hidden: true,
}
var (
osdCluster mon.ClusterInfo
osdCluster mon.ClusterInfo
osdDataDeviceFilter string
)
func addOSDFlags(command *cobra.Command) {
command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement")
command.Flags().StringVar(&cfg.devices, "data-devices", "", "comma separated list of devices to use for storage, or \"all\"")
command.Flags().StringVar(&cfg.devices, "data-devices", "", "comma separated list of devices to use for storage")
command.Flags().StringVar(&osdDataDeviceFilter, "data-device-filter", "", "a regex filter for the device names to use, or \"all\"")
command.Flags().StringVar(&cfg.directories, "data-directories", "", "comma separated list of directory paths to use for storage")
command.Flags().StringVar(&cfg.metadataDevice, "metadata-device", "", "device to use for metadata (e.g. a high performance SSD/NVMe device)")
command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement")
command.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
"true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!")
// bluestore config flags
command.Flags().IntVar(&cfg.bluestoreConfig.WalSizeMB, "osd-wal-size", partition.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL)")
command.Flags().IntVar(&cfg.bluestoreConfig.DatabaseSizeMB, "osd-database-size", partition.DBDefaultSizeMB, "default size (MB) for OSD database")
// OSD store config flags
command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osd.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)")
command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", osd.DBDefaultSizeMB, "default size (MB) for OSD database (bluestore)")
command.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osd.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)")
command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", osd.Filestore, "type of backing OSD store to use (bluestore or filestore)")
}
func init() {
@@ -60,13 +67,25 @@ func startOSD(cmd *cobra.Command, args []string) error {
return err
}
var dataDevices string
var usingDeviceFilter bool
if osdDataDeviceFilter != "" {
if cfg.devices != "" {
return fmt.Errorf("Only one of --data-devices and --data-device-filter can be specified.")
}
dataDevices = osdDataDeviceFilter
usingDeviceFilter = true
} else {
dataDevices = cfg.devices
}
setLogLevel()
metadataDevice := ""
forceFormat := false
location := ""
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
agent := osd.NewAgent(cephd.New(), cfg.devices, metadataDevice, forceFormat, location, cfg.bluestoreConfig, &clusterInfo)
agent := osd.NewAgent(cephd.New(), dataDevices, usingDeviceFilter, cfg.metadataDevice, cfg.directories, forceFormat,
cfg.location, cfg.storeConfig, &clusterInfo)
context := clusterd.NewDaemonContext(cfg.dataDir, cfg.cephConfigOverride, cfg.logLevel)
return osd.Run(context, agent)
Generated
+21 -8
View File
@@ -1,5 +1,5 @@
hash: 81178d6bc6321564bed3985f0746ccd7357067d5c9eaecfdccdbd86cfd897387
updated: 2017-03-08T10:56:46.619273095-08:00
hash: 23db89af71e3e11cfa7bb7722f8dd338f3285dd87270f8e6000898662e7def08
updated: 2017-04-03T13:42:53.811824499-07:00
imports:
- name: bitbucket.org/ww/goautoneg
version: a547fc61f48d567d5b4ec6f8aee5573d8efce11d
@@ -150,7 +150,7 @@ imports:
- name: github.com/inconshreveable/mousetrap
version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
- name: github.com/jonboulle/clockwork
version: 72f9bd7c4e0c2a40055ab3d0f09654f730cce982
version: 2eee05ed794112d45db504eb05aa693efd2b8b09
- name: github.com/juju/ratelimit
version: 77ed1c8a01217656d2080ad51981f6e99adaa177
- name: github.com/mailru/easyjson
@@ -198,13 +198,12 @@ imports:
- name: github.com/xiang90/probing
version: 07dd2e8dfe18522e9c447ba95f2fe95262f63bb2
- name: golang.org/x/crypto
version: 1f22c0103821b9390939b6776727195525381532
version: 1351f936d976c60a0a48d728281922cf63eafb8d
subpackages:
- bcrypt
- blowfish
- ssh/terminal
- name: golang.org/x/net
version: e90d6d0afc4c315a0d87a568ae68577cc15149a0
version: 6acef71eb69611914f7a30939ea9f6e194c78172
subpackages:
- context
- context/ctxhttp
@@ -212,7 +211,6 @@ imports:
- http2/hpack
- idna
- internal/timeseries
- lex/httplex
- trace
- name: golang.org/x/oauth2
version: 3c3a985cb79f52a3190fbc056984415ca6763d01
@@ -222,7 +220,7 @@ imports:
- jws
- jwt
- name: golang.org/x/sys
version: 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9
version: 9c60d1c508f5134d1ca726b4641db998f2523357
subpackages:
- unix
- name: golang.org/x/text
@@ -269,19 +267,33 @@ imports:
version: e121606b0d09b2e1c467183ee46217fa85a6b672
subpackages:
- discovery
- discovery/fake
- kubernetes
- kubernetes/fake
- kubernetes/typed/apps/v1beta1
- kubernetes/typed/apps/v1beta1/fake
- kubernetes/typed/authentication/v1beta1
- kubernetes/typed/authentication/v1beta1/fake
- kubernetes/typed/authorization/v1beta1
- kubernetes/typed/authorization/v1beta1/fake
- kubernetes/typed/autoscaling/v1
- kubernetes/typed/autoscaling/v1/fake
- kubernetes/typed/batch/v1
- kubernetes/typed/batch/v1/fake
- kubernetes/typed/batch/v2alpha1
- kubernetes/typed/batch/v2alpha1/fake
- kubernetes/typed/certificates/v1alpha1
- kubernetes/typed/certificates/v1alpha1/fake
- kubernetes/typed/core/v1
- kubernetes/typed/core/v1/fake
- kubernetes/typed/extensions/v1beta1
- kubernetes/typed/extensions/v1beta1/fake
- kubernetes/typed/policy/v1beta1
- kubernetes/typed/policy/v1beta1/fake
- kubernetes/typed/rbac/v1alpha1
- kubernetes/typed/rbac/v1alpha1/fake
- kubernetes/typed/storage/v1beta1
- kubernetes/typed/storage/v1beta1/fake
- pkg/api
- pkg/api/errors
- pkg/api/install
@@ -370,6 +382,7 @@ imports:
- plugin/pkg/client/auth/gcp
- plugin/pkg/client/auth/oidc
- rest
- testing
- tools/clientcmd/api
- tools/metrics
- transport
+1
View File
@@ -19,6 +19,7 @@ import:
- package: github.com/spf13/pflag
- package: github.com/google/uuid
- package: github.com/go-ini/ini
- package: github.com/ghodss/yaml
- package: github.com/gorilla/mux
- package: bitbucket.org/ww/goautoneg
version: a547fc61f48d567d5b4ec6f8aee5573d8efce11d
+3 -4
View File
@@ -20,7 +20,6 @@ import (
"github.com/rook/rook/pkg/cephmgr/mds"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/cephmgr/osd"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/cephmgr/rgw"
"github.com/rook/rook/pkg/clusterd"
)
@@ -30,15 +29,15 @@ const (
)
// create a new ceph service
func NewCephService(factory client.ConnectionFactory, devices, metadataDevice string, forceFormat bool,
location, adminSecret string, bluestoreConfig partition.BluestoreConfig) *clusterd.ClusterService {
func NewCephService(factory client.ConnectionFactory, devices, metadataDevice, directories string, forceFormat bool,
location, adminSecret string, storeConfig osd.StoreConfig) *clusterd.ClusterService {
return &clusterd.ClusterService{
Name: cephName,
Leader: newLeader(factory, adminSecret),
Agents: []clusterd.ServiceAgent{
mon.NewAgent(factory),
osd.NewAgent(factory, devices, metadataDevice, forceFormat, location, bluestoreConfig, nil),
osd.NewAgent(factory, devices, false, metadataDevice, directories, forceFormat, location, storeConfig, nil),
mds.NewAgent(factory),
rgw.NewAgent(factory),
},
+2 -2
View File
@@ -22,7 +22,7 @@ import (
testceph "github.com/rook/rook/pkg/cephmgr/client/test"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/cephmgr/osd"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
"github.com/rook/rook/pkg/util"
@@ -138,7 +138,7 @@ func TestRefreshKeys(t *testing.T) {
func TestNewCephService(t *testing.T) {
factory := &testceph.MockConnectionFactory{Fsid: "fsid", SecretKey: "key"}
service := NewCephService(factory, "a,b,c", "", true, "root=default", "", partition.BluestoreConfig{})
service := NewCephService(factory, "a,b,c", "", "", true, "root=default", "", osd.StoreConfig{})
assert.NotNil(t, service)
assert.Equal(t, "/rook/services/ceph/osd/desired", service.Leader.RefreshKeys()[0].Path)
assert.Equal(t, 4, len(service.Agents))
+94 -57
View File
@@ -35,7 +35,6 @@ import (
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/util"
"github.com/rook/rook/pkg/util/proc"
@@ -53,24 +52,27 @@ const (
)
type OsdAgent struct {
cluster *mon.ClusterInfo
forceFormat bool
location string
factory client.ConnectionFactory
osdProc map[int]*proc.MonitoredProc
desiredDevices []string
devices string
metadataDevice string
bluestoreConfig partition.BluestoreConfig
configCounter int32
osdsCompleted chan struct{}
cluster *mon.ClusterInfo
forceFormat bool
location string
factory client.ConnectionFactory
osdProc map[int]*proc.MonitoredProc
desiredDevices []string
desiredDirectories []string
devices string
usingDeviceFilter bool
metadataDevice string
directories string
storeConfig StoreConfig
configCounter int32
osdsCompleted chan struct{}
}
func NewAgent(factory client.ConnectionFactory, devices, metadataDevice string, forceFormat bool,
location string, bluestoreConfig partition.BluestoreConfig, cluster *mon.ClusterInfo) *OsdAgent {
func NewAgent(factory client.ConnectionFactory, devices string, usingDeviceFilter bool, metadataDevice, directories string, forceFormat bool,
location string, storeConfig StoreConfig, cluster *mon.ClusterInfo) *OsdAgent {
return &OsdAgent{factory: factory, devices: devices, metadataDevice: metadataDevice, forceFormat: forceFormat,
location: location, bluestoreConfig: bluestoreConfig, cluster: cluster}
return &OsdAgent{factory: factory, devices: devices, usingDeviceFilter: usingDeviceFilter, metadataDevice: metadataDevice,
directories: directories, forceFormat: forceFormat, location: location, storeConfig: storeConfig, cluster: cluster}
}
func (a *OsdAgent) Name() string {
@@ -86,13 +88,24 @@ func (a *OsdAgent) Initialize(context *clusterd.Context) error {
logger.Infof("desired devices for osds: %+v. desired metadata device: %s", a.desiredDevices, a.metadataDevice)
}
if len(a.directories) > 0 {
a.desiredDirectories = strings.Split(a.directories, ",")
}
// if no devices or directories were specified, use the current directory for an osd
if len(a.desiredDevices) == 0 {
if len(a.desiredDevices) == 0 && len(a.desiredDirectories) == 0 {
logger.Infof("Adding local path %s to desired state", context.ConfigDir)
err := AddDesiredDir(context.EtcdClient, context.ConfigDir, context.NodeID)
if err != nil {
return fmt.Errorf("failed to add current dir %s. %v", context.ConfigDir, err)
a.desiredDirectories = []string{context.ConfigDir}
}
if len(a.desiredDirectories) > 0 {
for _, dir := range a.desiredDirectories {
err := AddDesiredDir(context.EtcdClient, dir, context.NodeID)
if err != nil {
return fmt.Errorf("failed to add desired dir %s. %v", dir, err)
}
}
logger.Infof("desired directories for osds: %+v.", a.desiredDirectories)
}
return nil
@@ -351,7 +364,7 @@ func (a *OsdAgent) configureDirs(context *clusterd.Context, dirs map[string]int)
}
}
logger.Infof("%d/%d osds (filestore) succeeded on this node", succeeded, len(dirs))
logger.Infof("%d/%d osd dirs succeeded on this node", succeeded, len(dirs))
return lastErr
}
@@ -387,7 +400,7 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
defer a.decrementConfigCounter()
// compute an OSD layout scheme that will optimize performance
scheme, err := getPartitionPerfScheme(context, connection, devices, a.bluestoreConfig)
scheme, err := getPartitionPerfScheme(context, connection, devices, a.storeConfig)
logger.Debugf("partition scheme: %+v, err: %+v", scheme, err)
if err != nil {
logger.Errorf("failed to get OSD performance scheme: %+v", err)
@@ -396,8 +409,8 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
if scheme.Metadata != nil {
// partition the dedicated metadata device
if err := partitionBluestoreMetadata(context, scheme.Metadata, context.ConfigDir); err != nil {
logger.Errorf("failed to partition bluestore metadata %+v: %+v", scheme.Metadata, err)
if err := partitionMetadata(context, scheme.Metadata, context.ConfigDir); err != nil {
logger.Errorf("failed to partition metadata %+v: %+v", scheme.Metadata, err)
return
}
}
@@ -414,7 +427,7 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
}
}
logger.Infof("%d/%d bluestore osds succeeded on this node", succeeded, len(scheme.Entries))
logger.Infof("%d/%d osd devices succeeded on this node", succeeded, len(scheme.Entries))
}()
return nil
@@ -423,10 +436,10 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
// computes a partitioning scheme for all the given desired devices. This could be devics already in use,
// devices dedicated to metadata, and devices with all bluestore partitions collocated.
func getPartitionPerfScheme(context *clusterd.Context, connection client.Connection, devices *DeviceOsdMapping,
bluestoreConfig partition.BluestoreConfig) (*partition.PerfScheme, error) {
storeConfig StoreConfig) (*PerfScheme, error) {
// load the existing (committed) partition scheme from disk
perfScheme, err := partition.LoadScheme(context.ConfigDir)
perfScheme, err := LoadScheme(context.ConfigDir)
if err != nil {
return nil, fmt.Errorf("failed to load partition scheme from %s: %+v", context.ConfigDir, err)
}
@@ -461,7 +474,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
}
metadataEntry = mapping
perfScheme.Metadata = partition.NewMetadataDeviceInfo(name)
perfScheme.Metadata = NewMetadataDeviceInfo(name)
}
}
@@ -478,7 +491,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
return nil, fmt.Errorf("failed to register OSD for device %s: %+v", name, err)
}
schemeEntry := partition.NewPerfSchemeEntry()
schemeEntry := NewPerfSchemeEntry(storeConfig.StoreType)
schemeEntry.ID = *osdID
schemeEntry.OsdUUID = *osdUUID
@@ -488,7 +501,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
mapping.Data = *osdID
// populate the perf partition scheme entry with distributed partition details
err := partition.PopulateDistributedPerfSchemeEntry(schemeEntry, name, perfScheme.Metadata, bluestoreConfig)
err := PopulateDistributedPerfSchemeEntry(schemeEntry, name, perfScheme.Metadata, storeConfig)
if err != nil {
return nil, fmt.Errorf("failed to create distributed perf scheme entry for %s: %+v", name, err)
}
@@ -500,7 +513,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
mapping.Metadata = []int{*osdID}
// populate the perf partition scheme entry with collocated partition details
err := partition.PopulateCollocatedPerfSchemeEntry(schemeEntry, name, bluestoreConfig)
err := PopulateCollocatedPerfSchemeEntry(schemeEntry, name, storeConfig)
if err != nil {
return nil, fmt.Errorf("failed to create collocated perf scheme entry for %s: %+v", name, err)
}
@@ -514,7 +527,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
}
// determines if the given device name is already in use with existing/committed partitions
func isDeviceInUse(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) bool {
func isDeviceInUse(name string, nameToUUID map[string]string, scheme *PerfScheme) bool {
parts := findPartitionsForDevice(name, nameToUUID, scheme)
return len(parts) > 0
}
@@ -529,12 +542,12 @@ func isDeviceDesiredForData(mapping *DeviceOsdIDEntry) bool {
(mapping.Data > unassignedOSDID && len(mapping.Metadata) == 1)
}
func isDeviceDesiredForMetadata(mapping *DeviceOsdIDEntry, scheme *partition.PerfScheme) bool {
func isDeviceDesiredForMetadata(mapping *DeviceOsdIDEntry, scheme *PerfScheme) bool {
return mapping.Data == unassignedOSDID && mapping.Metadata != nil && len(mapping.Metadata) == 0
}
// finds all the partition details that are on the given device name
func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) []*partition.PerfSchemePartitionDetails {
func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *PerfScheme) []*PerfSchemePartitionDetails {
if scheme == nil {
return nil
}
@@ -544,7 +557,7 @@ func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *
return nil
}
parts := []*partition.PerfSchemePartitionDetails{}
parts := []*PerfSchemePartitionDetails{}
for _, e := range scheme.Entries {
for _, p := range e.Partitions {
if p.DiskUUID == diskUUID {
@@ -558,7 +571,7 @@ func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *
// if a device name has changed, this function will find all partition entries with the device's static UUID and
// then update the device name on them
func refreshDeviceInfo(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) {
func refreshDeviceInfo(name string, nameToUUID map[string]string, scheme *PerfScheme) {
parts := findPartitionsForDevice(name, nameToUUID, scheme)
if len(parts) == 0 {
return
@@ -595,7 +608,7 @@ func (a *OsdAgent) startOSD(context *clusterd.Context, connection client.Connect
if newOSD {
if config.partitionScheme != nil {
// format and partition the device if needed
savedScheme, err := partition.LoadScheme(config.configRoot)
savedScheme, err := LoadScheme(config.configRoot)
if err != nil {
return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", config.configRoot, err)
}
@@ -610,39 +623,25 @@ func (a *OsdAgent) startOSD(context *clusterd.Context, connection client.Connect
}
if !skipFormat {
err = formatDevice(context, config, a.forceFormat)
err = formatDevice(context, config, a.forceFormat, a.storeConfig)
if err != nil {
return fmt.Errorf("failed format/partition of osd %d. %+v", config.id, err)
}
logger.Notice("waiting after bluestore partition/format...")
logger.Notice("waiting after partition/format...")
<-time.After(2 * time.Second)
}
}
// osd_data_dir/whoami does not exist yet, create/initialize the OSD
err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, context.Executor)
err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, a.storeConfig, context.Executor)
if err != nil {
return fmt.Errorf("failed to initialize OSD at %s: %+v", config.rootPath, err)
}
// save the osd information to applied state
dataDiskUUID, metadataDiskUUID := "", ""
if config.partitionScheme != nil {
dataDiskUUID = config.partitionScheme.Partitions[partition.BlockPartitionName].DiskUUID
metadataDiskUUID = config.partitionScheme.Partitions[partition.DatabasePartitionName].DiskUUID
}
if context.EtcdClient != nil {
settings := map[string]string{
"path": config.configRoot,
dataDiskUUIDKey: dataDiskUUID,
metadataDiskUUIDKey: metadataDiskUUID,
}
key := path.Join(getAppliedKey(context.NodeID), fmt.Sprintf("%d", config.id))
if err := util.StoreEtcdProperties(context.EtcdClient, key, settings); err != nil {
return fmt.Errorf("failed to mark osd %d as applied: %+v", config.id, err)
}
if err := markOSDAsApplied(context, config); err != nil {
return fmt.Errorf("failed to mark osd %d as applied: %+v", config.id, err)
}
} else {
// osd_data_dir/whoami already exists, meaning the OSD is already set up.
@@ -680,7 +679,7 @@ func (a *OsdAgent) runOSD(context *clusterd.Context, clusterName string, config
osdUUIDArg,
}
if config.dir {
if !isBluestore(config) {
params = append(params, fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(config.rootPath)))
}
@@ -854,6 +853,40 @@ func associateOSDIDsWithMetadataDevice(etcdClient etcd.KeysAPI, nodeID, name, id
return err
}
func markOSDAsApplied(context *clusterd.Context, config *osdConfig) error {
if context.EtcdClient == nil {
return nil
}
dataDiskUUID, metadataDiskUUID := "", ""
if config.partitionScheme != nil {
dataPartDetails, err := getDataPartitionDetails(config)
if err != nil {
return err
}
metadataPartDetails, err := getMetadataPartitionDetails(config)
if err != nil {
return err
}
dataDiskUUID = dataPartDetails.DiskUUID
metadataDiskUUID = metadataPartDetails.DiskUUID
}
settings := map[string]string{
"path": config.configRoot,
dataDiskUUIDKey: dataDiskUUID,
metadataDiskUUIDKey: metadataDiskUUID,
}
key := path.Join(getAppliedKey(context.NodeID), fmt.Sprintf("%d", config.id))
if err := util.StoreEtcdProperties(context.EtcdClient, key, settings); err != nil {
return err
}
return nil
}
// add a device to the desired state
func AddDesiredDevice(etcdClient etcd.KeysAPI, nodeID, uuid string, osdID int) error {
key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), uuid)
@@ -954,3 +987,7 @@ func loadOSDInfo(config *osdConfig) error {
config.uuid = osdUUID
return nil
}
func isBluestore(config *osdConfig) bool {
return !config.dir && config.partitionScheme != nil && config.partitionScheme.StoreType == Bluestore
}
+77 -36
View File
@@ -28,7 +28,6 @@ import (
"github.com/google/uuid"
testceph "github.com/rook/rook/pkg/cephmgr/client/test"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
"github.com/rook/rook/pkg/util"
@@ -38,6 +37,11 @@ import (
)
func TestOSDAgentWithDevices(t *testing.T) {
testOSDAgentWithDevicesHelper(t, StoreConfig{StoreType: Bluestore})
testOSDAgentWithDevicesHelper(t, StoreConfig{StoreType: Filestore})
}
func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig StoreConfig) {
// set up a temporary config directory that will be cleaned up after test
configDir, err := ioutil.TempDir("", "TestOSDAgentWithDevices")
if err != nil {
@@ -47,7 +51,7 @@ func TestOSDAgentWithDevices(t *testing.T) {
clusterName := "mycluster"
nodeID := "abc"
etcdClient, agent, _ := createTestAgent(t, nodeID, "sdx,sdy", configDir)
etcdClient, agent, _ := createTestAgent(t, nodeID, "sdx,sdy", configDir, &storeConfig)
startCount := 0
executor := &exectest.MockExecutor{}
@@ -74,10 +78,10 @@ func TestOSDAgentWithDevices(t *testing.T) {
nameSuffix = parts[1]
}
switch {
case execCount == 0: // first exec is the mkfs for sdy
case execCount == 0: // first exec is the mkfs for sdx
assert.Equal(t, "--mkfs", args[3])
createTestKeyring(t, configDir, args)
case execCount == 1: // all remaining execs are for partitioning sdx then mkfs sdx
case execCount == 1: // all remaining execs are for partitioning sdy then mkfs sdy
assert.Equal(t, "sgdisk", command)
assert.Equal(t, "--zap-all", args[0])
assert.Equal(t, "/dev/"+nameSuffix, args[1])
@@ -86,10 +90,39 @@ func TestOSDAgentWithDevices(t *testing.T) {
assert.Equal(t, "--clear", args[0])
assert.Equal(t, "/dev/"+nameSuffix, args[2])
case execCount == 3:
assert.Equal(t, "/dev/"+nameSuffix, args[10])
// the partitioning for sdy. depending on bluestore/filestore, the args will be different.
expectedDeviceArgNum := 0
if storeConfig.StoreType == Bluestore {
expectedDeviceArgNum = 10
} else {
expectedDeviceArgNum = 4
}
assert.Equal(t, "sgdisk", command)
assert.Equal(t, "/dev/"+nameSuffix, args[expectedDeviceArgNum])
case execCount == 4:
assert.Equal(t, "--mkfs", args[3])
createTestKeyring(t, configDir, args)
if storeConfig.StoreType == Bluestore {
// the osd mkfs for sdy bluestore
assert.Equal(t, "--mkfs", args[3])
createTestKeyring(t, configDir, args)
} else {
// mkfs.ext4 for sdy filestore
assert.Equal(t, "mkfs.ext4", command)
}
case execCount == 5:
if storeConfig.StoreType == Filestore {
// the mount for sdy filestore
assert.Equal(t, "mount", command)
} else {
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
}
case execCount == 6:
if storeConfig.StoreType == Filestore {
// the osd mkfs for sdy filestore
assert.Equal(t, "--mkfs", args[3])
createTestKeyring(t, configDir, args)
} else {
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
}
default:
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
}
@@ -136,11 +169,16 @@ func TestOSDAgentWithDevices(t *testing.T) {
<-agent.osdsCompleted
assert.Equal(t, 0, agent.configCounter)
assert.Equal(t, 5, execCount) // 1 mkfs for sdy, 3 partition steps for sdx, 1 mkfs for sdx
assert.Equal(t, 2, outputExecCount)
assert.Equal(t, 2, startCount) // 2 OSD procs should be started
assert.Equal(t, 2, len(agent.osdProc), fmt.Sprintf("procs=%+v", agent.osdProc))
if storeConfig.StoreType == Bluestore {
assert.Equal(t, 5, execCount) // 1 osd mkfs for sdx, 3 partition steps for sdy, 1 osd mkfs for sdy
} else {
assert.Equal(t, 7, execCount) // 1 osd mkfs for sdx, 3 partition steps for sdy, 1 mkfs for sdy, 1 mount for sdy, 1 osd mkfs for sdy
}
err = agent.DestroyLocalService(context)
assert.Nil(t, err)
assert.Equal(t, 0, len(agent.osdProc))
@@ -159,7 +197,7 @@ func TestOSDAgentNoDevices(t *testing.T) {
// create a test OSD agent with no devices specified
nodeID := "abc"
etcdClient, agent, _ := createTestAgent(t, nodeID, "", configDir)
etcdClient, agent, _ := createTestAgent(t, nodeID, "", configDir, nil)
startCount := 0
executor := &exectest.MockExecutor{}
@@ -251,7 +289,7 @@ func TestRemoveDevice(t *testing.T) {
defer os.RemoveAll(configDir)
nodeID := "a"
etcdClient, agent, conn := createTestAgent(t, nodeID, "", configDir)
etcdClient, agent, conn := createTestAgent(t, nodeID, "", configDir, nil)
executor := &exectest.MockExecutor{}
context := &clusterd.Context{EtcdClient: etcdClient, NodeID: nodeID, Executor: executor, ProcMan: proc.New(executor), Inventory: createInventory()}
@@ -272,12 +310,15 @@ func TestRemoveDevice(t *testing.T) {
assert.True(t, applied.Equals(util.CreateSet([]string{"23"})), fmt.Sprintf("applied=%+v", applied))
}
func createTestAgent(t *testing.T, nodeID, devices, configDir string) (*util.MockEtcdClient, *OsdAgent, *testceph.MockConnection) {
func createTestAgent(t *testing.T, nodeID, devices, configDir string, storeConfig *StoreConfig) (*util.MockEtcdClient, *OsdAgent, *testceph.MockConnection) {
location := "root=here"
forceFormat := false
if storeConfig == nil {
storeConfig = &StoreConfig{StoreType: Bluestore}
}
etcdClient := util.NewMockEtcdClient()
factory := &testceph.MockConnectionFactory{}
agent := NewAgent(factory, devices, "", forceFormat, location, partition.BluestoreConfig{}, nil)
agent := NewAgent(factory, devices, false, "", "", forceFormat, location, *storeConfig, nil)
agent.cluster = &mon.ClusterInfo{Name: "myclust"}
agent.Initialize(&clusterd.Context{EtcdClient: etcdClient, NodeID: nodeID, ConfigDir: configDir})
if devices == "" {
@@ -443,7 +484,7 @@ func TestGetPartitionPerfScheme(t *testing.T) {
return nil, "", fmt.Errorf("unexpected mon_command '%s'", string(args))
}
scheme, err := getPartitionPerfScheme(context, conn, devices, partition.BluestoreConfig{})
scheme, err := getPartitionPerfScheme(context, conn, devices, StoreConfig{})
assert.Nil(t, err)
assert.Equal(t, 2, len(scheme.Entries))
@@ -457,11 +498,11 @@ func TestGetPartitionPerfScheme(t *testing.T) {
// and that the second is the other one.
entry := scheme.Entries[0]
assert.Equal(t, 11, entry.ID)
firstBlockDevice := entry.Partitions[partition.BlockPartitionName].Device
firstBlockDevice := entry.Partitions[BlockPartitionType].Device
assert.True(t, firstBlockDevice == "sda" || firstBlockDevice == "sdb", firstBlockDevice)
verifyPartitionEntry(t, entry.Partitions[partition.BlockPartitionName], firstBlockDevice, -1, 1)
verifyPartitionEntry(t, entry.Partitions[partition.WalPartitionName], "sdc", partition.WalDefaultSizeMB, 1)
verifyPartitionEntry(t, entry.Partitions[partition.DatabasePartitionName], "sdc", partition.DBDefaultSizeMB, 577)
verifyPartitionEntry(t, entry.Partitions[BlockPartitionType], firstBlockDevice, -1, 1)
verifyPartitionEntry(t, entry.Partitions[WalPartitionType], "sdc", WalDefaultSizeMB, 1)
verifyPartitionEntry(t, entry.Partitions[DatabasePartitionType], "sdc", DBDefaultSizeMB, 577)
// verify the second entry in the scheme. Note the comment above about sda vs. sdb ordering.
entry = scheme.Entries[1]
@@ -472,9 +513,9 @@ func TestGetPartitionPerfScheme(t *testing.T) {
} else {
secondBlockDevice = "sda"
}
verifyPartitionEntry(t, entry.Partitions[partition.BlockPartitionName], secondBlockDevice, -1, 1)
verifyPartitionEntry(t, entry.Partitions[partition.WalPartitionName], "sdc", partition.WalDefaultSizeMB, 21057)
verifyPartitionEntry(t, entry.Partitions[partition.DatabasePartitionName], "sdc", partition.DBDefaultSizeMB, 21633)
verifyPartitionEntry(t, entry.Partitions[BlockPartitionType], secondBlockDevice, -1, 1)
verifyPartitionEntry(t, entry.Partitions[WalPartitionType], "sdc", WalDefaultSizeMB, 21057)
verifyPartitionEntry(t, entry.Partitions[DatabasePartitionType], "sdc", DBDefaultSizeMB, 21633)
}
func TestGetPartitionPerfSchemeDiskInUse(t *testing.T) {
@@ -508,7 +549,7 @@ func TestGetPartitionPerfSchemeDiskInUse(t *testing.T) {
// get the partition scheme based on the desired devices. Since sda is already in use, the partition
// scheme returned should reflect that.
scheme, err := getPartitionPerfScheme(context, nil, devices, partition.BluestoreConfig{})
scheme, err := getPartitionPerfScheme(context, nil, devices, StoreConfig{})
assert.Nil(t, err)
// the partition scheme should have a single entry for osd 1 on sda and it should have collocated data and metadata
@@ -553,20 +594,20 @@ func TestGetPartitionPerfSchemeDiskNameChanged(t *testing.T) {
// get the current partition scheme. This should notice that the device names changed and update the
// partition scheme to have the latest device names
scheme, err := getPartitionPerfScheme(context, nil, devices, partition.BluestoreConfig{})
scheme, err := getPartitionPerfScheme(context, nil, devices, StoreConfig{})
assert.Nil(t, err)
assert.NotNil(t, scheme)
assert.Equal(t, "nvme01-changed", scheme.Metadata.Device)
assert.Equal(t, "sda-changed", scheme.Entries[0].Partitions[partition.BlockPartitionName].Device)
assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[partition.WalPartitionName].Device)
assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[partition.DatabasePartitionName].Device)
assert.Equal(t, "sda-changed", scheme.Entries[0].Partitions[BlockPartitionType].Device)
assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[WalPartitionType].Device)
assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[DatabasePartitionType].Device)
}
func createInventory() *inventory.Config {
return &inventory.Config{Local: &inventory.Hardware{Disks: []*inventory.LocalDisk{}}}
}
func verifyPartitionEntry(t *testing.T, actual *partition.PerfSchemePartitionDetails, expectedDevice string,
func verifyPartitionEntry(t *testing.T, actual *PerfSchemePartitionDetails, expectedDevice string,
expectedSize int64, expectedOffset int64) {
assert.Equal(t, expectedDevice, actual.Device)
@@ -574,12 +615,12 @@ func verifyPartitionEntry(t *testing.T, actual *partition.PerfSchemePartitionDet
assert.Equal(t, expectedOffset, actual.OffsetMB)
}
func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string) (entry *partition.PerfSchemeEntry, diskUUID string) {
entry = partition.NewPerfSchemeEntry()
func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string) (entry *PerfSchemeEntry, diskUUID string) {
entry = NewPerfSchemeEntry(Bluestore)
entry.ID = osdID
entry.OsdUUID = uuid.Must(uuid.NewRandom())
partition.PopulateCollocatedPerfSchemeEntry(entry, device, partition.BluestoreConfig{})
scheme := partition.NewPerfScheme()
PopulateCollocatedPerfSchemeEntry(entry, device, StoreConfig{})
scheme := NewPerfScheme()
scheme.Entries = append(scheme.Entries, entry)
err := scheme.Save(configDir)
assert.Nil(t, err)
@@ -594,19 +635,19 @@ func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string)
return entry, diskUUID
}
func mockDistributedPartitionScheme(t *testing.T, osdID int, metadataDevice, device, configDir string) (*partition.PerfScheme, string, string) {
scheme := partition.NewPerfScheme()
scheme.Metadata = partition.NewMetadataDeviceInfo(metadataDevice)
func mockDistributedPartitionScheme(t *testing.T, osdID int, metadataDevice, device, configDir string) (*PerfScheme, string, string) {
scheme := NewPerfScheme()
scheme.Metadata = NewMetadataDeviceInfo(metadataDevice)
entry := partition.NewPerfSchemeEntry()
entry := NewPerfSchemeEntry(Bluestore)
entry.ID = osdID
entry.OsdUUID = uuid.Must(uuid.NewRandom())
partition.PopulateDistributedPerfSchemeEntry(entry, device, scheme.Metadata, partition.BluestoreConfig{})
PopulateDistributedPerfSchemeEntry(entry, device, scheme.Metadata, StoreConfig{})
scheme.Entries = append(scheme.Entries, entry)
err := scheme.Save(configDir)
assert.Nil(t, err)
// return the full partition scheme, the metadata device UUID and the data device UUID
return scheme, scheme.Metadata.DiskUUID, entry.Partitions[partition.BlockPartitionName].DiskUUID
return scheme, scheme.Metadata.DiskUUID, entry.Partitions[BlockPartitionType].DiskUUID
}
+2 -2
View File
@@ -49,14 +49,14 @@ func formatLocation(location string) ([]string, error) {
}
}
if !isCrushFieldSet("hostName", pairs) {
if !isCrushFieldSet("host", pairs) {
// host name isn't set yet, attempt to set a default
hostName, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("failed to get hostname, %+v", err)
}
pairs = append(pairs, formatProperty("hostName", hostName))
pairs = append(pairs, formatProperty("host", hostName))
}
// set a default root if it's not already set
+5 -5
View File
@@ -49,12 +49,12 @@ func TestCrushMap(t *testing.T) {
argsSet := util.CreateSet(request.Args)
assert.True(t, argsSet.Contains("root=default"))
assert.True(t, argsSet.Contains("dc=datacenter1"))
assert.True(t, argsSet.Contains("hostName=node1"))
assert.True(t, argsSet.Contains("host=node1"))
return []byte{}, "", nil
}
location := "root=default,dc=datacenter1,hostName=node1"
location := "root=default,dc=datacenter1,host=node1"
err := addOSDToCrushMap(conn, context, 23, "/", location)
assert.Nil(t, err)
@@ -84,17 +84,17 @@ func TestCrushLocation(t *testing.T) {
locSet := util.CreateSet(res)
assert.True(t, locSet.Contains("root=default"))
assert.True(t, locSet.Contains("dc=datacenter1"))
assert.True(t, locSet.Contains(fmt.Sprintf("hostName=%s", hostName)))
assert.True(t, locSet.Contains(fmt.Sprintf("host=%s", hostName)))
// test that if host name and root are already set they will be honored
loc = "root=otherRoot,dc=datacenter2,hostName=node123"
loc = "root=otherRoot,dc=datacenter2,host=node123"
res, err = formatLocation(loc)
assert.Nil(t, err)
assert.Equal(t, 3, len(res))
locSet = util.CreateSet(res)
assert.True(t, locSet.Contains("root=otherRoot"))
assert.True(t, locSet.Contains("dc=datacenter2"))
assert.True(t, locSet.Contains("hostName=node123"))
assert.True(t, locSet.Contains("host=node123"))
// test an invalid CRUSH location format
loc = "root=default,prop:value"
+68 -19
View File
@@ -25,6 +25,8 @@ import (
"regexp"
"time"
"strings"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
@@ -62,7 +64,7 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
}
// initialize the desired osds
devices, err := getAvailableDevices(dcontext, hardware.Disks, agent.devices)
devices, err := getAvailableDevices(dcontext, hardware.Disks, agent.devices, agent.usingDeviceFilter)
if err != nil {
return fmt.Errorf("failed to get available devices. %+v", err)
}
@@ -74,8 +76,8 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
}
// initialize the data directories, with the default dir if no devices were specified
useDataDir := len(devices.Entries) == 0
dirs, err := getDataDirs(dcontext, useDataDir)
devicesSpecified := len(devices.Entries) > 0
dirs, err := getDataDirs(dcontext, agent.directories, devicesSpecified)
if err != nil {
return fmt.Errorf("failed to get data dirs. %+v", err)
}
@@ -95,7 +97,14 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
return nil
}
func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk, desiredDevices string) (*DeviceOsdMapping, error) {
func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk, desiredDevices string,
usingDeviceFilter bool) (*DeviceOsdMapping, error) {
var deviceList []string
if !usingDeviceFilter {
deviceList = strings.Split(desiredDevices, ",")
}
available := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}}
for _, device := range devices {
if device.Type == sys.PartType {
@@ -109,12 +118,24 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L
if desiredDevices == "all" {
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
} else if desiredDevices != "" {
// the desired devices is a regular expression
matched, err := regexp.Match(desiredDevices, []byte(device.Name))
var matched bool
var err error
if usingDeviceFilter {
// the desired devices is a regular expression
matched, err = regexp.Match(desiredDevices, []byte(device.Name))
} else {
for i := range deviceList {
if device.Name == deviceList[i] {
matched = true
break
}
}
}
if err == nil && matched {
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
} else {
logger.Infof("skipping device %s that does not match the regular expression %s. %+v", device.Name, desiredDevices, err)
logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err)
}
} else {
@@ -126,28 +147,56 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L
return available, nil
}
func getDataDirs(context *clusterd.DaemonContext, useDataDir bool) (map[string]int, error) {
filePath := path.Join(context.ConfigDir, dirOSDConfigFilename)
if _, err := os.Stat(filePath); os.IsNotExist(err) {
// the config file doesn't exist yet, just return the empty map unless the data dir is desired
if useDataDir {
return map[string]int{context.ConfigDir: unassignedOSDID}, nil
}
return map[string]int{}, nil
func getDataDirs(context *clusterd.DaemonContext, desiredDirs string, devicesSpecified bool) (map[string]int, error) {
var dirList []string
if desiredDirs != "" {
dirList = strings.Split(desiredDirs, ",")
}
filePath := path.Join(context.ConfigDir, dirOSDConfigFilename)
if _, err := os.Stat(filePath); os.IsNotExist(err) {
// the config file doesn't exist yet
if len(dirList) == 0 {
if devicesSpecified {
// no dirs desired, user is using devices instead
return map[string]int{}, nil
} else {
// no devices or dirs specified, return the default data dir
return map[string]int{context.ConfigDir: unassignedOSDID}, nil
}
}
dirMap := make(map[string]int, len(dirList))
addDirsToDirMap(dirList, &dirMap)
return dirMap, nil
}
// read the saved dir map from disk
var dirMap map[string]int
b, err := ioutil.ReadFile(filePath)
if err != nil {
return nil, err
}
var dirs map[string]int
err = json.Unmarshal(b, &dirs)
err = json.Unmarshal(b, &dirMap)
if err != nil {
return nil, err
}
return dirs, nil
// if the user has specified any directories to use, merge them into the saved config
addDirsToDirMap(dirList, &dirMap)
return dirMap, nil
}
func addDirsToDirMap(dirList []string, dirMap *map[string]int) {
for _, d := range dirList {
if _, ok := (*dirMap)[d]; !ok {
// the users dir isn't already in the map, add it with an unassigned ID
(*dirMap)[d] = unassignedOSDID
}
}
}
func saveDirConfig(context *clusterd.DaemonContext, config map[string]int) error {
+36 -14
View File
@@ -35,33 +35,51 @@ func TestStoreOSDDirMap(t *testing.T) {
defer os.RemoveAll(context.ConfigDir)
os.MkdirAll(context.ConfigDir, 0755)
dirMap, err := getDataDirs(context, false)
// user has specified devices to use, no dirs should be returned
dirMap, err := getDataDirs(context, "", true)
assert.Nil(t, err)
assert.Equal(t, 0, len(dirMap))
dirMap, err = getDataDirs(context, true)
// user has no devices specified, should return default dir
dirMap, err = getDataDirs(context, "", false)
assert.Nil(t, err)
assert.Equal(t, 1, len(dirMap))
assert.Equal(t, unassignedOSDID, dirMap[context.ConfigDir])
dirMap[context.ConfigDir] = 0
// user has no devices specified but does specify dirs, those should be returned
dirMap, err = getDataDirs(context, "/rook/dir1", false)
assert.Nil(t, err)
assert.Equal(t, 1, len(dirMap))
assert.Equal(t, unassignedOSDID, dirMap["/rook/dir1"])
dirMap["/rook/dir1"] = 0 // simulate an OSD ID being assigned to the dir
// save the directory config
err = saveDirConfig(context, dirMap)
assert.Nil(t, err)
dirMap, err = getDataDirs(context, false)
// user has specified devices to use, we should still return the saved dir
dirMap, err = getDataDirs(context, "", true)
assert.Nil(t, err)
assert.Equal(t, 1, len(dirMap))
assert.Equal(t, 0, dirMap[context.ConfigDir])
assert.Equal(t, 0, dirMap["/rook/dir1"])
// add another directory to the map
// user has specified devices and also a directory to use. it should be added to the dir map
dirMap, err = getDataDirs(context, "/tmp/mydir", true)
assert.Nil(t, err)
assert.Equal(t, 2, len(dirMap))
assert.Equal(t, 0, dirMap["/rook/dir1"])
assert.Equal(t, unassignedOSDID, dirMap["/tmp/mydir"])
// simulate that the user's dir got an OSD by assigning it an ID
dirMap["/tmp/mydir"] = 23
err = saveDirConfig(context, dirMap)
assert.Nil(t, err)
dirMap, err = getDataDirs(context, false)
// user is still specifying the directory, we should get back it's ID now
dirMap, err = getDataDirs(context, "/tmp/mydir", true)
assert.Nil(t, err)
assert.Equal(t, 2, len(dirMap))
assert.Equal(t, 0, dirMap[context.ConfigDir])
assert.Equal(t, 0, dirMap["/rook/dir1"])
assert.Equal(t, 23, dirMap["/tmp/mydir"])
}
@@ -99,7 +117,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="MY-PART"`, nil
&inventory.LocalDisk{Name: "rdb"},
}
// select all devices
mapping, err := getAvailableDevices(context, devices, "all")
mapping, err := getAvailableDevices(context, devices, "all", true)
assert.Nil(t, err)
assert.Equal(t, 4, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sda"].Data)
@@ -107,26 +125,30 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="MY-PART"`, nil
assert.Equal(t, -1, mapping.Entries["rda"].Data)
assert.Equal(t, -1, mapping.Entries["rdb"].Data)
// select no devices
mapping, err = getAvailableDevices(context, devices, "")
// select no devices both using and not using a filter
mapping, err = getAvailableDevices(context, devices, "", false)
assert.Nil(t, err)
assert.Equal(t, 0, len(mapping.Entries))
mapping, err = getAvailableDevices(context, devices, "", true)
assert.Nil(t, err)
assert.Equal(t, 0, len(mapping.Entries))
// select the sda devices
mapping, err = getAvailableDevices(context, devices, "^sd.$")
mapping, err = getAvailableDevices(context, devices, "^sd.$", true)
assert.Nil(t, err)
assert.Equal(t, 2, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sda"].Data)
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
// select an exact device
mapping, err = getAvailableDevices(context, devices, "sdd")
mapping, err = getAvailableDevices(context, devices, "sdd", false)
assert.Nil(t, err)
assert.Equal(t, 1, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
// select all devices except those that have a prefix of "s"
mapping, err = getAvailableDevices(context, devices, "^[^s]")
mapping, err = getAvailableDevices(context, devices, "^[^s]", true)
assert.Nil(t, err)
assert.Equal(t, 2, len(mapping.Entries))
assert.Equal(t, -1, mapping.Entries["rda"].Data)
+125 -50
View File
@@ -25,12 +25,12 @@ import (
"strconv"
"strings"
"syscall"
"time"
"github.com/google/uuid"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
"github.com/rook/rook/pkg/util"
@@ -41,6 +41,7 @@ import (
const (
DevicesValue = "devices"
ForceFormatValue = "forceFormat"
diskByPartUUID = "/dev/disk/by-partuuid"
cephOsdKey = mon.CephKey + "/osd"
desiredOsdRootKey = cephOsdKey + "/" + clusterd.DesiredKey + "/%s"
deviceDesiredKey = desiredOsdRootKey + "/device"
@@ -58,7 +59,7 @@ type osdConfig struct {
id int
uuid uuid.UUID
dir bool
partitionScheme *partition.PerfSchemeEntry
partitionScheme *PerfSchemeEntry
}
type Device struct {
@@ -83,42 +84,42 @@ type DeviceOsdIDEntry struct {
}
// format the given device for usage by an OSD
func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool) error {
blockDetails, err := getBlockPartitionDetails(config)
func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool, storeConfig StoreConfig) error {
dataDetails, err := getDataPartitionDetails(config)
if err != nil {
return err
}
// check if partitions belong to rook
ownPartitions, devFS, err := checkIfDeviceAvailable(context.Executor, blockDetails.Device)
ownPartitions, devFS, err := checkIfDeviceAvailable(context.Executor, dataDetails.Device)
if err != nil {
return fmt.Errorf("failed to format device. %+v", err)
}
if !ownPartitions {
if forceFormat {
logger.Warningf("device %s is being formatted, but has partitions!!", blockDetails.Device)
logger.Warningf("device %s is being formatted, but has partitions!!", dataDetails.Device)
} else {
logger.Warningf("device %s has partitions that will not be formatted. Skipping device.", blockDetails.Device)
logger.Warningf("device %s has partitions that will not be formatted. Skipping device.", dataDetails.Device)
}
}
if devFS != "" {
if forceFormat {
// there's a filesystem on the device, but the user has specified to force a format. give a warning about that.
logger.Warningf("device %s already formatted with %s, but forcing a format!!!", blockDetails.Device, devFS)
logger.Warningf("device %s already formatted with %s, but forcing a format!!!", dataDetails.Device, devFS)
} else {
// disk is already formatted and the user doesn't want to force it, but we require partitioning
return fmt.Errorf("device %s already formatted with %s", blockDetails.Device, devFS)
return fmt.Errorf("device %s already formatted with %s", dataDetails.Device, devFS)
}
}
// format the device
dangerousToFormat := !ownPartitions || devFS != ""
if !dangerousToFormat || forceFormat {
err := partitionBluestoreOSD(context, config)
err := partitionOSD(context, config)
if err != nil {
return fmt.Errorf("failed to partion device %s. %v", blockDetails.Device, err)
return fmt.Errorf("failed to partion device %s. %v", dataDetails.Device, err)
}
}
@@ -157,14 +158,14 @@ func rookOwnsPartitions(partitions []*sys.Partition) bool {
return true
}
// partitions a given device exclusively for bluestore metadata usage
func partitionBluestoreMetadata(context *clusterd.Context, info *partition.MetadataDeviceInfo, configRoot string) error {
// partitions a given device exclusively for metadata usage
func partitionMetadata(context *clusterd.Context, info *MetadataDeviceInfo, configRoot string) error {
if len(info.Partitions) == 0 {
return nil
}
// check to see if the metadata partition scheme has already been applied
savedScheme, err := partition.LoadScheme(configRoot)
savedScheme, err := LoadScheme(configRoot)
if err != nil {
return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", configRoot, err)
}
@@ -212,28 +213,53 @@ func partitionBluestoreMetadata(context *clusterd.Context, info *partition.Metad
return nil
}
// Partitions a device for use by a bluestore osd.
// Partitions a device for use by a osd.
// If there are any partitions or formatting already on the device, it will be wiped.
func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error {
blockDetails, err := getBlockPartitionDetails(config)
func partitionOSD(context *clusterd.Context, config *osdConfig) error {
dataDetails, err := getDataPartitionDetails(config)
if err != nil {
return err
}
// zap/clear all existing partitions on the device
err = sys.RemovePartitions(blockDetails.Device, context.Executor)
err = sys.RemovePartitions(dataDetails.Device, context.Executor)
if err != nil {
return fmt.Errorf("failed to zap partitions on metadata device /dev/%s: %+v", blockDetails.Device, err)
return fmt.Errorf("failed to zap partitions on metadata device /dev/%s: %+v", dataDetails.Device, err)
}
// create the partitions on the device
err = sys.CreatePartitions(blockDetails.Device, config.partitionScheme.GetPartitionArgs(), context.Executor)
err = sys.CreatePartitions(dataDetails.Device, config.partitionScheme.GetPartitionArgs(), context.Executor)
if err != nil {
return fmt.Errorf("failed to partition /dev/%s. %+v", blockDetails.Device, err)
return fmt.Errorf("failed to partition /dev/%s. %+v", dataDetails.Device, err)
}
if config.partitionScheme.StoreType == Filestore {
// the OSD is using filestore, create a filesystem for the device and mount it under config root
dataPartDetails := config.partitionScheme.Partitions[FilestoreDataPartitionType]
dataPartPath := filepath.Join(diskByPartUUID, dataPartDetails.PartitionUUID)
logger.Infof("waiting for partition path %s", dataPartPath)
err = waitForPath(dataPartPath, context.Executor)
if err != nil {
return fmt.Errorf("failed waiting for %s: %+v", dataPartPath, err)
}
err = sys.FormatDevice(dataPartPath, context.Executor)
if err != nil {
logger.Warningf("first attempt to format partition %s on device %s failed. Waiting 2 seconds then retrying: %+v",
dataPartDetails.PartitionUUID, dataPartDetails.Device, err)
<-time.After(2 * time.Second)
err = sys.FormatDevice(dataPartPath, context.Executor)
if err != nil {
return fmt.Errorf("failed to format partition %s on device %s. %+v", dataPartDetails.PartitionUUID, dataPartDetails.Device, err)
}
}
err = sys.MountDevice(dataPartPath, config.rootPath, context.Executor)
if err != nil {
return fmt.Errorf("failed to mount %s at %s: %+v", dataPartPath, config.configRoot, context.Executor)
}
}
// save the partition scheme entry to disk now that it has been committed
savedScheme, err := partition.LoadScheme(config.configRoot)
savedScheme, err := LoadScheme(config.configRoot)
if err != nil {
return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", config.configRoot, err)
}
@@ -243,27 +269,27 @@ func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error {
}
// update the uuid of the disk in the inventory in memory
logger.Debugf("Updating disk uuid %s on device %s", blockDetails.DiskUUID, blockDetails.Device)
logger.Debugf("Updating disk uuid %s on device %s", dataDetails.DiskUUID, dataDetails.Device)
for _, disk := range context.Inventory.Local.Disks {
if disk.Name == blockDetails.Device {
logger.Debugf("Updated uuid on device %s", blockDetails.Device)
disk.UUID = blockDetails.DiskUUID
if disk.Name == dataDetails.Device {
logger.Debugf("Updated uuid on device %s", dataDetails.Device)
disk.UUID = dataDetails.DiskUUID
}
}
// save the desired state of the osd for this device
if context.EtcdClient != nil {
err = associateOsdIDWithDevice(context.EtcdClient, context.NodeID, blockDetails.DiskUUID, config.id, false)
err = associateOsdIDWithDevice(context.EtcdClient, context.NodeID, dataDetails.DiskUUID, config.id, false)
if err != nil {
return fmt.Errorf("failed to associate osd id %d with device %s (%s)", config.id, blockDetails.Device, blockDetails.DiskUUID)
return fmt.Errorf("failed to associate osd id %d with device %s (%s)", config.id, dataDetails.Device, dataDetails.DiskUUID)
}
if config.partitionScheme.IsCollocated() {
// the metadata is on the same disk as the data, associate the osd ID with the device for metadata too
err = associateOSDIDsWithMetadataDevice(
context.EtcdClient, context.NodeID, blockDetails.DiskUUID, fmt.Sprintf("%d", config.id))
context.EtcdClient, context.NodeID, dataDetails.DiskUUID, fmt.Sprintf("%d", config.id))
if err != nil {
return fmt.Errorf("failed to associate osd id %d with device %s (%s) for metadata",
config.id, blockDetails.Device, blockDetails.DiskUUID)
config.id, dataDetails.Device, dataDetails.DiskUUID)
}
}
}
@@ -271,17 +297,39 @@ func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error {
return nil
}
func getBlockPartitionDetails(config *osdConfig) (*partition.PerfSchemePartitionDetails, error) {
func getDataPartitionDetails(config *osdConfig) (*PerfSchemePartitionDetails, error) {
if config.partitionScheme == nil {
return nil, fmt.Errorf("partition scheme missing from %+v", config)
}
blockDetails, ok := config.partitionScheme.Partitions[partition.BlockPartitionName]
if !ok || blockDetails == nil {
return nil, fmt.Errorf("block partition missing from %+v", config.partitionScheme)
dataPartitionType := config.partitionScheme.getDataPartitionType()
dataDetails, ok := config.partitionScheme.Partitions[dataPartitionType]
if !ok || dataDetails == nil {
return nil, fmt.Errorf("data partition missing from %+v", config.partitionScheme)
}
return blockDetails, nil
return dataDetails, nil
}
func getMetadataPartitionDetails(config *osdConfig) (*PerfSchemePartitionDetails, error) {
if config.partitionScheme == nil {
return nil, fmt.Errorf("partition scheme missing from %+v", config)
}
metadataPartitionType := config.partitionScheme.getMetadataPartitionType()
if config.partitionScheme.StoreType == Filestore {
// TODO: support separate metadata device for filestore (just use the data partition details for now)
return getDataPartitionDetails(config)
}
metadataDetails, ok := config.partitionScheme.Partitions[metadataPartitionType]
if !ok || metadataDetails == nil {
return nil, fmt.Errorf("metadata partition missing from %+v", config.partitionScheme)
}
return metadataDetails, nil
}
func getDiskSize(context *clusterd.Context, name string) (uint64, error) {
@@ -294,6 +342,24 @@ func getDiskSize(context *clusterd.Context, name string) (uint64, error) {
return 0, fmt.Errorf("device %s not found", name)
}
func waitForPath(path string, executor exec.Executor) error {
retryCount := 0
retryMax := 25
sleepTime := 250
for {
_, err := executor.ExecuteStat(path)
if err == nil {
return nil
}
retryCount++
if retryCount > retryMax {
return err
}
<-time.After(time.Duration(sleepTime) * time.Millisecond)
}
}
func registerOSD(bootstrapConn client.Connection) (*int, *uuid.UUID, error) {
var err error
osdUUID, err := uuid.NewRandom()
@@ -311,9 +377,15 @@ func registerOSD(bootstrapConn client.Connection) (*int, *uuid.UUID, error) {
return &osdID, &osdUUID, nil
}
func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]string, error) {
func getStoreSettings(context *clusterd.Context, config *osdConfig, storeConfig StoreConfig) (map[string]string, error) {
settings := map[string]string{}
if config.dir {
if config.dir || (config.partitionScheme != nil && config.partitionScheme.StoreType == Filestore) {
// add additional filestore settings for dirs and filestore devices
journalSize := JournalDefaultSizeMB
if storeConfig.JournalSizeMB > 0 {
journalSize = storeConfig.JournalSizeMB
}
settings["osd journal size"] = strconv.Itoa(journalSize)
return settings, nil
}
@@ -321,21 +393,22 @@ func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]
return nil, fmt.Errorf("failed to find partitions from config for osd %d", config.id)
}
// add additional bluestore settings
parts := config.partitionScheme.Partitions
walPartition, ok := parts[partition.WalPartitionName]
walPartition, ok := parts[WalPartitionType]
if !ok {
return nil, fmt.Errorf("failed to find wal partition for osd %d", config.id)
}
dbPartition, ok := parts[partition.DatabasePartitionName]
dbPartition, ok := parts[DatabasePartitionType]
if !ok {
return nil, fmt.Errorf("failed to find db partition for osd %d", config.id)
}
blockPartition, ok := parts[partition.BlockPartitionName]
blockPartition, ok := parts[BlockPartitionType]
if !ok {
return nil, fmt.Errorf("failed to find block partition for osd %d", config.id)
}
prefix := "/dev/disk/by-partuuid"
prefix := diskByPartUUID
settings["bluestore block wal path"] = path.Join(prefix, walPartition.PartitionUUID)
settings["bluestore block db path"] = path.Join(prefix, dbPartition.PartitionUUID)
settings["bluestore block path"] = path.Join(prefix, blockPartition.PartitionUUID)
@@ -343,12 +416,12 @@ func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]
return settings, nil
}
func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context,
bootstrapConn client.Connection, cluster *mon.ClusterInfo, location string, executor exec.Executor) error {
func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context, bootstrapConn client.Connection,
cluster *mon.ClusterInfo, location string, storeConfig StoreConfig, executor exec.Executor) error {
cephConfig := mon.CreateDefaultCephConfig(context, cluster, config.rootPath, !config.dir)
cephConfig := mon.CreateDefaultCephConfig(context, cluster, config.rootPath, isBluestore(config))
if config.dir {
if !isBluestore(config) {
// using the local file system requires some config overrides
// http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/#not-recommended
cephConfig.GlobalConfig.OsdMaxObjectNameLen = 256
@@ -356,7 +429,7 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context
}
// bluestore has some extra settings
settings, err := getStoreSettings(context, config)
settings, err := getStoreSettings(context, config, storeConfig)
if err != nil {
return fmt.Errorf("failed to read store settings. %+v", err)
}
@@ -364,7 +437,7 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context
// write the OSD config file to disk
keyringPath := getOSDKeyringPath(config.rootPath)
_, err = mon.GenerateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id),
keyringPath, !config.dir, cephConfig, settings)
keyringPath, isBluestore(config), cephConfig, settings)
if err != nil {
return fmt.Errorf("failed to write OSD %d config file: %+v", config.id, err)
}
@@ -461,7 +534,7 @@ func createOSDFileSystem(context *clusterd.Context, clusterName string, config *
fmt.Sprintf("--monmap=%s", monMapTmpPath),
}
if config.dir {
if !isBluestore(config) {
options = append(options, fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(config.rootPath)))
options = append(options, fmt.Sprintf("--keyring=%s", getOSDKeyringPath(config.rootPath)))
}
@@ -523,8 +596,6 @@ func addOSDToCrushMap(osdConn client.Connection, context *clusterd.Context, osdI
weight, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", weight), 64)
osdEntity := fmt.Sprintf("osd.%d", osdID)
logger.Infof("OSD %s at %s, bytes: %d, weight: %.4f", osdEntity, osdDataPath, all, weight)
locArgs, err := formatLocation(location)
if err != nil {
return err
@@ -536,6 +607,10 @@ func addOSDToCrushMap(osdConn client.Connection, context *clusterd.Context, osdI
"weight": weight,
"args": locArgs,
}
logger.Infof("adding %s (%s), bytes: %d, weight: %.4f, to crush map at '%s'",
osdEntity, osdDataPath, all, weight, strings.Join(locArgs, " "))
_, err = client.ExecuteMonCommand(osdConn, cmd, fmt.Sprintf("adding %s to crush map", osdEntity))
if err != nil {
return fmt.Errorf("failed adding %s to crush map: %+v", osdEntity, err)
+54 -25
View File
@@ -25,7 +25,6 @@ import (
"github.com/google/uuid"
testceph "github.com/rook/rook/pkg/cephmgr/client/test"
"github.com/rook/rook/pkg/cephmgr/osd/partition"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/clusterd/inventory"
"github.com/rook/rook/pkg/util"
@@ -133,11 +132,13 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda" PARTLABEL="ROOK-OSD0-BLOCK"`
return output, nil
}
storeConfig := StoreConfig{StoreType: Bluestore}
// set up a partition scheme entry for sda (collocated metadata and data)
entry := partition.NewPerfSchemeEntry()
entry := NewPerfSchemeEntry(storeConfig.StoreType)
entry.ID = 1
entry.OsdUUID = uuid.Must(uuid.NewRandom())
partition.PopulateCollocatedPerfSchemeEntry(entry, "sda", partition.BluestoreConfig{})
PopulateCollocatedPerfSchemeEntry(entry, "sda", storeConfig)
context := &clusterd.Context{EtcdClient: etcdClient, Executor: executor, NodeID: nodeID,
ConfigDir: configDir, Inventory: createInventory()}
@@ -154,7 +155,7 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda" PARTLABEL="ROOK-OSD0-BLOCK"`
// try to format the device. even though the device has existing partitions, they are owned by rook, so it is safe
// to format and the format/partitioning will happen.
err = formatDevice(context, config, false)
err = formatDevice(context, config, false, storeConfig)
assert.Nil(t, err)
assert.Equal(t, 3, execCount)
assert.Equal(t, 3, outputExecCount)
@@ -195,21 +196,21 @@ func TestPartitionBluestoreMetadata(t *testing.T) {
context := &clusterd.Context{EtcdClient: etcdClient, Executor: executor, NodeID: nodeID, ConfigDir: configDir}
// create metadata partition information for 2 OSDs (sdb, sdc) storing their metadata on device sda
bluestoreConfig := partition.BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
metadata := partition.NewMetadataDeviceInfo("sda")
storeConfig := StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2}
metadata := NewMetadataDeviceInfo("sda")
e1 := partition.NewPerfSchemeEntry()
e1 := NewPerfSchemeEntry(Bluestore)
e1.ID = 1
e1.OsdUUID = uuid.Must(uuid.NewRandom())
partition.PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, bluestoreConfig)
PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, storeConfig)
e2 := partition.NewPerfSchemeEntry()
e2 := NewPerfSchemeEntry(Bluestore)
e2.ID = 2
e2.OsdUUID = uuid.Must(uuid.NewRandom())
partition.PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, bluestoreConfig)
PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, storeConfig)
// perform the metadata device partition
err = partitionBluestoreMetadata(context, metadata, configDir)
err = partitionMetadata(context, metadata, configDir)
assert.Nil(t, err)
assert.Equal(t, 3, execCount)
@@ -221,7 +222,12 @@ func TestPartitionBluestoreMetadata(t *testing.T) {
assert.True(t, util.CreateSet(desiredIds).Equals(util.CreateSet([]string{"1", "2"})))
}
func TestPartitionBluestoreOSD(t *testing.T) {
func TestPartitionOSD(t *testing.T) {
testPartitionOSDHelper(t, StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2})
testPartitionOSDHelper(t, StoreConfig{StoreType: Filestore})
}
func testPartitionOSDHelper(t *testing.T, storeConfig StoreConfig) {
// set up a temporary config directory that will be cleaned up after test
configDir, err := ioutil.TempDir("", "TestPartitionBluestoreOSD")
if err != nil {
@@ -237,17 +243,36 @@ func TestPartitionBluestoreOSD(t *testing.T) {
executor := &exectest.MockExecutor{}
executor.MockExecuteCommand = func(name string, command string, args ...string) error {
logger.Infof("RUN %d for '%s'. %s %+v", execCount, name, command, args)
assert.Equal(t, "sgdisk", command)
if execCount <= 2 {
assert.Equal(t, "sgdisk", command)
}
switch execCount {
case 0:
assert.Equal(t, []string{"--zap-all", "/dev/sda"}, args)
case 1:
assert.Equal(t, []string{"--clear", "--mbrtogpt", "/dev/sda"}, args)
case 2:
assert.Equal(t, 11, len(args))
assert.Equal(t, "--change-name=1:ROOK-OSD1-WAL", args[1])
assert.Equal(t, "--change-name=2:ROOK-OSD1-DB", args[4])
assert.Equal(t, "--change-name=3:ROOK-OSD1-BLOCK", args[7])
if storeConfig.StoreType == Bluestore {
assert.Equal(t, 11, len(args))
assert.Equal(t, "--change-name=1:ROOK-OSD1-WAL", args[1])
assert.Equal(t, "--change-name=2:ROOK-OSD1-DB", args[4])
assert.Equal(t, "--change-name=3:ROOK-OSD1-BLOCK", args[7])
} else {
assert.Equal(t, 5, len(args))
assert.Equal(t, "--change-name=1:ROOK-OSD1-FS-DATA", args[1])
}
case 3:
if storeConfig.StoreType == Bluestore {
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
} else {
assert.Equal(t, "mkfs.ext4", command)
}
case 4:
if storeConfig.StoreType == Bluestore {
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
} else {
assert.Equal(t, "mount", command)
}
}
execCount++
return nil
@@ -260,28 +285,32 @@ func TestPartitionBluestoreOSD(t *testing.T) {
}
// setup a partition scheme for data and metadata to be collocated on sda
bluestoreConfig := partition.BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
entry := partition.NewPerfSchemeEntry()
entry := NewPerfSchemeEntry(storeConfig.StoreType)
entry.ID = 1
entry.OsdUUID = uuid.Must(uuid.NewRandom())
partition.PopulateCollocatedPerfSchemeEntry(entry, "sda", bluestoreConfig)
PopulateCollocatedPerfSchemeEntry(entry, "sda", storeConfig)
config := &osdConfig{configRoot: configDir, rootPath: filepath.Join(configDir, "osd1"), id: entry.ID,
uuid: entry.OsdUUID, dir: false, partitionScheme: entry}
// partition the OSD on sda now
err = partitionBluestoreOSD(context, config)
err = partitionOSD(context, config)
assert.Nil(t, err)
assert.Equal(t, 3, execCount)
if storeConfig.StoreType == Bluestore {
assert.Equal(t, 3, execCount)
} else {
assert.Equal(t, 5, execCount)
}
// verify that both the data and metadata have been associated with the device in etcd (since data/metadata are collocated)
blockDetails, err := getBlockPartitionDetails(config)
dataDetails, err := getDataPartitionDetails(config)
assert.Nil(t, err)
dataID := etcdClient.GetValue(
fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-data", blockDetails.DiskUUID))
fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-data", dataDetails.DiskUUID))
assert.Equal(t, "1", dataID)
metadataID := etcdClient.GetValue(
fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-metadata", blockDetails.DiskUUID))
fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-metadata", dataDetails.DiskUUID))
assert.Equal(t, "1", metadataID)
}
@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package partition
package osd
import (
"encoding/json"
@@ -26,21 +26,31 @@ import (
)
const (
WalPartitionName = "wal"
DatabasePartitionName = "db"
BlockPartitionName = "block"
walPartition = 1
databasePartition = 2
blockPartition = 3
UseRemainingSpace = -1
schemeFilename = "partition-scheme"
WalDefaultSizeMB = 576
DBDefaultSizeMB = 20480
Filestore = "filestore"
Bluestore = "bluestore"
DefaultStore = Filestore
UseRemainingSpace = -1
schemeFilename = "partition-scheme"
WalDefaultSizeMB = 576
DBDefaultSizeMB = 20480
JournalDefaultSizeMB = 5120
)
type BluestoreConfig struct {
WalSizeMB int
DatabaseSizeMB int
type PartitionType int
const (
WalPartitionType PartitionType = iota
DatabasePartitionType
BlockPartitionType
FilestoreDataPartitionType
FilestoreJournalPartitionType
)
type StoreConfig struct {
StoreType string `json:"storeType,omitempty"`
WalSizeMB int `json:"walSizeMB,omitempty"`
DatabaseSizeMB int `json:"databaseSizeMB,omitempty"`
JournalSizeMB int `json:"journalSizeMB,omitempty"`
}
// top level representation of an overall performance oriented partition scheme, with a dedicated metadata device
@@ -52,9 +62,10 @@ type PerfScheme struct {
// represents an OSD and details about all of its partitions
type PerfSchemeEntry struct {
ID int `json:"id"`
OsdUUID uuid.UUID `json:"osdUuid"`
Partitions map[string]*PerfSchemePartitionDetails `json:"partitions"` // mapping of partition name to its details
ID int `json:"id"`
OsdUUID uuid.UUID `json:"osdUuid"`
Partitions map[PartitionType]*PerfSchemePartitionDetails `json:"partitions"` // mapping of partition name to its details
StoreType string `json:"storeType,omitempty"`
}
// details for 1 OSD partition
@@ -75,12 +86,12 @@ type MetadataDeviceInfo struct {
// representsa specific partition on a metadata device, including details about which OSD it belongs to
type MetadataDevicePartition struct {
ID int `json:"id"`
OsdUUID uuid.UUID `json:"osdUuid"`
Name string `json:"name"`
PartitionUUID string `json:"partitionUuid"`
SizeMB int `json:"sizeMB"`
OffsetMB int `json:"offsetMB"`
ID int `json:"id"`
OsdUUID uuid.UUID `json:"osdUuid"`
Type PartitionType `json:"type"`
PartitionUUID string `json:"partitionUuid"`
SizeMB int `json:"sizeMB"`
OffsetMB int `json:"offsetMB"`
}
func NewPerfScheme() *PerfScheme {
@@ -89,9 +100,10 @@ func NewPerfScheme() *PerfScheme {
}
}
func NewPerfSchemeEntry() *PerfSchemeEntry {
func NewPerfSchemeEntry(storeType string) *PerfSchemeEntry {
return &PerfSchemeEntry{
Partitions: map[string]*PerfSchemePartitionDetails{}, // mapping of partition name (e.g. WAL) to it's details
Partitions: map[PartitionType]*PerfSchemePartitionDetails{}, // mapping of partition name (e.g. WAL) to it's details
StoreType: storeType,
}
}
@@ -137,48 +149,66 @@ func (s *PerfScheme) Save(configDir string) error {
}
// populates a partition scheme entry for an OSD where all its partitions are collocated on a single device
func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, bluestoreConfig BluestoreConfig) error {
diskUUID, walUUID, dbUUID, blockUUID, err := createUUIDs()
if err != nil {
return err
}
func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, storeConfig StoreConfig) error {
walSize := WalDefaultSizeMB
if bluestoreConfig.WalSizeMB > 0 {
walSize = bluestoreConfig.WalSizeMB
}
dbSize := DBDefaultSizeMB
if bluestoreConfig.DatabaseSizeMB > 0 {
dbSize = bluestoreConfig.DatabaseSizeMB
}
if storeConfig.StoreType == Filestore {
diskUUID, dataUUID, _, err := createFilestoreUUIDs()
if err != nil {
return err
}
offset := 1
// the filestore data partition will take up the entire given device (and we do not create a separate partition/entry
// for the journal)
entry.Partitions[FilestoreDataPartitionType] = &PerfSchemePartitionDetails{
Device: device,
DiskUUID: diskUUID.String(),
PartitionUUID: dataUUID.String(),
SizeMB: UseRemainingSpace,
OffsetMB: 1,
}
} else {
diskUUID, walUUID, dbUUID, blockUUID, err := createBluestoreUUIDs()
if err != nil {
return err
}
// layout the partitions for WAL, DB, and Block
entry.Partitions[WalPartitionName] = &PerfSchemePartitionDetails{
Device: device,
DiskUUID: diskUUID.String(),
PartitionUUID: walUUID.String(),
SizeMB: walSize,
OffsetMB: offset,
}
offset += entry.Partitions[WalPartitionName].SizeMB
walSize := WalDefaultSizeMB
if storeConfig.WalSizeMB > 0 {
walSize = storeConfig.WalSizeMB
}
dbSize := DBDefaultSizeMB
if storeConfig.DatabaseSizeMB > 0 {
dbSize = storeConfig.DatabaseSizeMB
}
entry.Partitions[DatabasePartitionName] = &PerfSchemePartitionDetails{
Device: device,
DiskUUID: diskUUID.String(),
PartitionUUID: dbUUID.String(),
SizeMB: dbSize,
OffsetMB: offset,
}
offset += entry.Partitions[DatabasePartitionName].SizeMB
offset := 1
entry.Partitions[BlockPartitionName] = &PerfSchemePartitionDetails{
Device: device,
DiskUUID: diskUUID.String(),
PartitionUUID: blockUUID.String(),
SizeMB: UseRemainingSpace,
OffsetMB: offset,
// layout the partitions for WAL, DB, and Block
entry.Partitions[WalPartitionType] = &PerfSchemePartitionDetails{
Device: device,
DiskUUID: diskUUID.String(),
PartitionUUID: walUUID.String(),
SizeMB: walSize,
OffsetMB: offset,
}
offset += entry.Partitions[WalPartitionType].SizeMB
entry.Partitions[DatabasePartitionType] = &PerfSchemePartitionDetails{
Device: device,
DiskUUID: diskUUID.String(),
PartitionUUID: dbUUID.String(),
SizeMB: dbSize,
OffsetMB: offset,
}
offset += entry.Partitions[DatabasePartitionType].SizeMB
entry.Partitions[BlockPartitionType] = &PerfSchemePartitionDetails{
Device: device,
DiskUUID: diskUUID.String(),
PartitionUUID: blockUUID.String(),
SizeMB: UseRemainingSpace,
OffsetMB: offset,
}
}
return nil
@@ -187,15 +217,20 @@ func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, bl
// populates a partition scheme entry for an OSD that will have distributed partitions: its metadata will live on a
// dedicated metadata device and its block data will live on a dedicated device
func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, metadataInfo *MetadataDeviceInfo,
bluestoreConfig BluestoreConfig) error {
storeConfig StoreConfig) error {
diskUUID, walUUID, dbUUID, blockUUID, err := createUUIDs()
if storeConfig.StoreType == Filestore {
// TODO: support separate metadata device for filestore
return fmt.Errorf("filestore not yet supported for distributed partition scheme")
}
diskUUID, walUUID, dbUUID, blockUUID, err := createBluestoreUUIDs()
if err != nil {
return err
}
// the block partition will take up the entire given device
entry.Partitions[BlockPartitionName] = &PerfSchemePartitionDetails{
entry.Partitions[BlockPartitionType] = &PerfSchemePartitionDetails{
Device: device,
DiskUUID: diskUUID.String(),
PartitionUUID: blockUUID.String(),
@@ -219,16 +254,16 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m
}
walSize := WalDefaultSizeMB
if bluestoreConfig.WalSizeMB > 0 {
walSize = bluestoreConfig.WalSizeMB
if storeConfig.WalSizeMB > 0 {
walSize = storeConfig.WalSizeMB
}
dbSize := DBDefaultSizeMB
if bluestoreConfig.DatabaseSizeMB > 0 {
dbSize = bluestoreConfig.DatabaseSizeMB
if storeConfig.DatabaseSizeMB > 0 {
dbSize = storeConfig.DatabaseSizeMB
}
// record information about the WAL partition
entry.Partitions[WalPartitionName] = &PerfSchemePartitionDetails{
entry.Partitions[WalPartitionType] = &PerfSchemePartitionDetails{
Device: metadataInfo.Device,
DiskUUID: metadataInfo.DiskUUID,
PartitionUUID: walUUID.String(),
@@ -238,16 +273,16 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m
walPartitionInfo := &MetadataDevicePartition{
ID: entry.ID,
OsdUUID: entry.OsdUUID,
Name: WalPartitionName,
Type: WalPartitionType,
PartitionUUID: walUUID.String(),
SizeMB: walSize,
OffsetMB: offset,
}
metadataInfo.Partitions = append(metadataInfo.Partitions, walPartitionInfo)
offset += entry.Partitions[WalPartitionName].SizeMB
offset += entry.Partitions[WalPartitionType].SizeMB
// record information about the DB partition
entry.Partitions[DatabasePartitionName] = &PerfSchemePartitionDetails{
entry.Partitions[DatabasePartitionType] = &PerfSchemePartitionDetails{
Device: metadataInfo.Device,
DiskUUID: metadataInfo.DiskUUID,
PartitionUUID: dbUUID.String(),
@@ -257,7 +292,7 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m
dbPartitionInfo := &MetadataDevicePartition{
ID: entry.ID,
OsdUUID: entry.OsdUUID,
Name: DatabasePartitionName,
Type: DatabasePartitionType,
PartitionUUID: dbUUID.String(),
SizeMB: dbSize,
OffsetMB: offset,
@@ -271,7 +306,7 @@ func (m *MetadataDeviceInfo) GetPartitionArgs() []string {
args := []string{}
for partNum, part := range m.Partitions {
partArgs := getPartitionArgs(partNum+1, part.PartitionUUID, part.OffsetMB, part.SizeMB, getPartitionLabel(part.ID, part.Name))
partArgs := getPartitionArgs(partNum+1, part.PartitionUUID, part.OffsetMB, part.SizeMB, getPartitionLabel(part.ID, part.Type))
args = append(args, partArgs...)
}
@@ -288,26 +323,28 @@ func (e *PerfSchemeEntry) GetPartitionArgs() []string {
args := []string{}
partNum := 1
if collocated {
if collocated && e.StoreType == Bluestore {
// partitions are collocated, create the metadata partitions on the same device
walDetails := e.Partitions[WalPartitionName]
partArgs := getPartitionArgsFromDetails(partNum, WalPartitionName, e.ID, walDetails)
walDetails := e.Partitions[WalPartitionType]
partArgs := getPartitionArgsFromDetails(partNum, WalPartitionType, e.ID, walDetails)
args = append(args, partArgs...)
partNum++
dbDetails := e.Partitions[DatabasePartitionName]
partArgs = getPartitionArgsFromDetails(partNum, DatabasePartitionName, e.ID, dbDetails)
dbDetails := e.Partitions[DatabasePartitionType]
partArgs = getPartitionArgsFromDetails(partNum, DatabasePartitionType, e.ID, dbDetails)
args = append(args, partArgs...)
partNum++
}
dataPartitionType := e.getDataPartitionType()
// always create the data partition
blockDetails := e.Partitions[BlockPartitionName]
partArgs := getPartitionArgsFromDetails(partNum, BlockPartitionName, e.ID, blockDetails)
args = append(args, partArgs...)
dataDetails := e.Partitions[dataPartitionType]
dataPartArgs := getPartitionArgsFromDetails(partNum, dataPartitionType, e.ID, dataDetails)
args = append(args, dataPartArgs...)
// append args for the whole device
args = append(args, []string{fmt.Sprintf("--disk-guid=%s", blockDetails.DiskUUID), "/dev/" + blockDetails.Device}...)
args = append(args, []string{fmt.Sprintf("--disk-guid=%s", dataDetails.DiskUUID), "/dev/" + dataDetails.Device}...)
return args
}
@@ -329,10 +366,26 @@ func (e *PerfSchemeEntry) IsCollocated() bool {
return collocated
}
func (e *PerfSchemeEntry) getDataPartitionType() PartitionType {
if e.StoreType == Filestore {
return FilestoreDataPartitionType
} else {
return BlockPartitionType
}
}
func (e *PerfSchemeEntry) getMetadataPartitionType() PartitionType {
if e.StoreType == Filestore {
return FilestoreJournalPartitionType
} else {
return DatabasePartitionType
}
}
// Get the arguments necessary to create an sgdisk partition with the given parameters.
// number is the partition number.
// The offset and length are in MB. Under the covers this is translated to sectors.
// If the length is -1, all remaining space will be assigned to the partition.
// If the length is -1, all remaining space will be assigned to the
func getPartitionArgs(number int, guid string, offset, length int, label string) []string {
const sectorsPerMB = 2048
var newPart string
@@ -351,24 +404,28 @@ func getPartitionArgs(number int, guid string, offset, length int, label string)
}
}
func getPartitionArgsFromDetails(number int, name string, id int, details *PerfSchemePartitionDetails) []string {
return getPartitionArgs(number, details.PartitionUUID, details.OffsetMB, details.SizeMB, getPartitionLabel(id, name))
func getPartitionArgsFromDetails(number int, partType PartitionType, id int, details *PerfSchemePartitionDetails) []string {
return getPartitionArgs(number, details.PartitionUUID, details.OffsetMB, details.SizeMB, getPartitionLabel(id, partType))
}
func getPartitionLabel(id int, partName string) string {
switch partName {
case WalPartitionName:
func getPartitionLabel(id int, partType PartitionType) string {
switch partType {
case WalPartitionType:
return fmt.Sprintf("ROOK-OSD%d-WAL", id)
case DatabasePartitionName:
case DatabasePartitionType:
return fmt.Sprintf("ROOK-OSD%d-DB", id)
case BlockPartitionName:
case BlockPartitionType:
return fmt.Sprintf("ROOK-OSD%d-BLOCK", id)
case FilestoreDataPartitionType:
return fmt.Sprintf("ROOK-OSD%d-FS-DATA", id)
case FilestoreJournalPartitionType:
return fmt.Sprintf("ROOK-OSD%d-FS-JOURNAL", id)
}
return ""
}
func createUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
func createBluestoreUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
diskUUID, err := uuid.NewRandom()
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("failed to get disk uuid. %+v", err)
@@ -391,3 +448,22 @@ func createUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
return &diskUUID, &walUUID, &dbUUID, &blockUUID, nil
}
func createFilestoreUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
diskUUID, err := uuid.NewRandom()
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to get disk uuid. %+v", err)
}
dataUUID, err := uuid.NewRandom()
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to get data uuid. %+v", err)
}
journalUUID, err := uuid.NewRandom()
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to get journal uuid. %+v", err)
}
return &diskUUID, &dataUUID, &journalUUID, nil
}
@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package partition
package osd
import (
"fmt"
@@ -43,15 +43,15 @@ func TestSchemeSaveLoad(t *testing.T) {
// add some entries to the scheme
scheme.Metadata = NewMetadataDeviceInfo("sda")
scheme.Metadata.DiskUUID = uuid.Must(uuid.NewRandom()).String()
m1 := &MetadataDevicePartition{ID: 1, OsdUUID: uuid.Must(uuid.NewRandom()), Name: "wal",
m1 := &MetadataDevicePartition{ID: 1, OsdUUID: uuid.Must(uuid.NewRandom()), Type: WalPartitionType,
PartitionUUID: uuid.Must(uuid.NewRandom()).String(), SizeMB: 100, OffsetMB: 1}
m2 := &MetadataDevicePartition{ID: 1, OsdUUID: m1.OsdUUID, Name: "db",
m2 := &MetadataDevicePartition{ID: 1, OsdUUID: m1.OsdUUID, Type: DatabasePartitionType,
PartitionUUID: uuid.Must(uuid.NewRandom()).String(), SizeMB: 200, OffsetMB: 101}
scheme.Metadata.Partitions = append(scheme.Metadata.Partitions, []*MetadataDevicePartition{m1, m2}...)
e1 := &PerfSchemeEntry{ID: 1, OsdUUID: m1.OsdUUID}
e1.Partitions = map[string]*PerfSchemePartitionDetails{
"block": &PerfSchemePartitionDetails{
e1.Partitions = map[PartitionType]*PerfSchemePartitionDetails{
BlockPartitionType: &PerfSchemePartitionDetails{
Device: "sdb",
DiskUUID: uuid.Must(uuid.NewRandom()).String(),
PartitionUUID: uuid.Must(uuid.NewRandom()).String(),
@@ -71,45 +71,45 @@ func TestSchemeSaveLoad(t *testing.T) {
}
func TestPopulateCollocatedPerfSchemeEntry(t *testing.T) {
entry := NewPerfSchemeEntry()
entry := NewPerfSchemeEntry(Bluestore)
entry.ID = 10
entry.OsdUUID = uuid.Must(uuid.NewRandom())
err := PopulateCollocatedPerfSchemeEntry(entry, "sda", BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2})
err := PopulateCollocatedPerfSchemeEntry(entry, "sda", StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2})
assert.Nil(t, err)
// verify the populated collocated partition entries
assert.Equal(t, 3, len(entry.Partitions))
verifyPartitionDetails(t, entry, WalPartitionName, "sda", 1, 1)
verifyPartitionDetails(t, entry, DatabasePartitionName, "sda", 2, 2)
verifyPartitionDetails(t, entry, BlockPartitionName, "sda", 4, -1)
verifyPartitionDetails(t, entry, WalPartitionType, "sda", 1, 1)
verifyPartitionDetails(t, entry, DatabasePartitionType, "sda", 2, 2)
verifyPartitionDetails(t, entry, BlockPartitionType, "sda", 4, -1)
}
func TestPopulateDistributedPerfSchemeEntry(t *testing.T) {
metadata := NewMetadataDeviceInfo("sda")
entry := NewPerfSchemeEntry()
entry := NewPerfSchemeEntry(Bluestore)
entry.ID = 20
entry.OsdUUID = uuid.Must(uuid.NewRandom())
err := PopulateDistributedPerfSchemeEntry(entry, "sdb", metadata, BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2})
err := PopulateDistributedPerfSchemeEntry(entry, "sdb", metadata, StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2})
assert.Nil(t, err)
// verify the populated distributed partition entries (metadata partitions should be on metadata device, block
// partition should be on its own device)
assert.Equal(t, 3, len(entry.Partitions))
verifyPartitionDetails(t, entry, WalPartitionName, "sda", 1, 1)
verifyPartitionDetails(t, entry, DatabasePartitionName, "sda", 2, 2)
verifyPartitionDetails(t, entry, BlockPartitionName, "sdb", 1, -1)
verifyPartitionDetails(t, entry, WalPartitionType, "sda", 1, 1)
verifyPartitionDetails(t, entry, DatabasePartitionType, "sda", 2, 2)
verifyPartitionDetails(t, entry, BlockPartitionType, "sdb", 1, -1)
// verify that the metadata device info was populated as well
assert.Equal(t, 2, len(metadata.Partitions))
verifyMetadataDevicePartition(t, metadata, 0, entry.ID, entry.OsdUUID, WalPartitionName, 1, 1)
verifyMetadataDevicePartition(t, metadata, 1, entry.ID, entry.OsdUUID, DatabasePartitionName, 2, 2)
verifyMetadataDevicePartition(t, metadata, 0, entry.ID, entry.OsdUUID, WalPartitionType, 1, 1)
verifyMetadataDevicePartition(t, metadata, 1, entry.ID, entry.OsdUUID, DatabasePartitionType, 2, 2)
}
func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partName, device string, offset, size int64) {
part, ok := entry.Partitions[partName]
func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partType PartitionType, device string, offset, size int64) {
part, ok := entry.Partitions[partType]
assert.True(t, ok)
assert.NotNil(t, part)
assert.Equal(t, device, part.Device)
@@ -118,13 +118,13 @@ func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partName, devi
}
func verifyMetadataDevicePartition(t *testing.T, info *MetadataDeviceInfo, index int, osdID int, osdUUID uuid.UUID,
name string, offset, size int64) {
partType PartitionType, offset, size int64) {
part := info.Partitions[index]
assert.NotNil(t, part)
assert.Equal(t, osdID, part.ID)
assert.Equal(t, osdUUID, part.OsdUUID)
assert.Equal(t, name, part.Name)
assert.Equal(t, partType, part.Type)
assert.Equal(t, offset, part.OffsetMB)
assert.Equal(t, size, part.SizeMB)
}
@@ -132,18 +132,18 @@ func verifyMetadataDevicePartition(t *testing.T, info *MetadataDeviceInfo, index
func TestMetadataGetPartitionArgs(t *testing.T) {
metadata := NewMetadataDeviceInfo("sda")
e1 := NewPerfSchemeEntry()
e1 := NewPerfSchemeEntry(Bluestore)
e1.ID = 1
e1.OsdUUID = uuid.Must(uuid.NewRandom())
e2 := NewPerfSchemeEntry()
e2 := NewPerfSchemeEntry(Bluestore)
e2.ID = 2
e2.OsdUUID = uuid.Must(uuid.NewRandom())
bluestoreConfig := BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
err := PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, bluestoreConfig)
storeConfig := StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
err := PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, storeConfig)
assert.Nil(t, err)
err = PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, bluestoreConfig)
err = PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, storeConfig)
assert.Nil(t, err)
expectedArgs := []string{
@@ -160,19 +160,19 @@ func TestMetadataGetPartitionArgs(t *testing.T) {
}
func TestSchemeEntryGetPartitionArgs(t *testing.T) {
e1 := NewPerfSchemeEntry()
e1 := NewPerfSchemeEntry(Bluestore)
e1.ID = 1
e1.OsdUUID = uuid.Must(uuid.NewRandom())
bluestoreConfig := BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
err := PopulateCollocatedPerfSchemeEntry(e1, "sdb", bluestoreConfig)
storeConfig := StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
err := PopulateCollocatedPerfSchemeEntry(e1, "sdb", storeConfig)
assert.Nil(t, err)
expectedArgs := []string{
"--new=1:2048:+2048", "--change-name=1:ROOK-OSD1-WAL", fmt.Sprintf("--partition-guid=1:%s", e1.Partitions[WalPartitionName].PartitionUUID),
"--new=2:4096:+4096", "--change-name=2:ROOK-OSD1-DB", fmt.Sprintf("--partition-guid=2:%s", e1.Partitions[DatabasePartitionName].PartitionUUID),
"--largest-new=3", "--change-name=3:ROOK-OSD1-BLOCK", fmt.Sprintf("--partition-guid=3:%s", e1.Partitions[BlockPartitionName].PartitionUUID),
fmt.Sprintf("--disk-guid=%s", e1.Partitions[BlockPartitionName].DiskUUID), "/dev/sdb",
"--new=1:2048:+2048", "--change-name=1:ROOK-OSD1-WAL", fmt.Sprintf("--partition-guid=1:%s", e1.Partitions[WalPartitionType].PartitionUUID),
"--new=2:4096:+4096", "--change-name=2:ROOK-OSD1-DB", fmt.Sprintf("--partition-guid=2:%s", e1.Partitions[DatabasePartitionType].PartitionUUID),
"--largest-new=3", "--change-name=3:ROOK-OSD1-BLOCK", fmt.Sprintf("--partition-guid=3:%s", e1.Partitions[BlockPartitionType].PartitionUUID),
fmt.Sprintf("--disk-guid=%s", e1.Partitions[BlockPartitionType].DiskUUID), "/dev/sdb",
}
// get the partition args and verify against expected
+1 -2
View File
@@ -99,8 +99,7 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
func (c *Cluster) apiContainer() v1.Container {
command := fmt.Sprintf("/usr/bin/rookd api --data-dir=%s --port=%d --container-version=%s",
k8sutil.DataDir, model.Port, c.Version)
command := fmt.Sprintf("/usr/bin/rookd api --config-dir=%s --port=%d --container-version=%s", k8sutil.DataDir, model.Port, c.Version)
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
+2 -2
View File
@@ -78,8 +78,8 @@ func TestPodSpecs(t *testing.T) {
for _, v := range cont.Env {
assert.True(t, strings.HasPrefix(v.Name, "ROOKD_"))
}
expectedCommand := fmt.Sprintf("/usr/bin/rookd api --data-dir=/var/lib/rook --port=%d --container-version=%s",
model.Port, c.Version)
expectedCommand := fmt.Sprintf("/usr/bin/rookd api --config-dir=/var/lib/rook --port=%d --container-version=%s", model.Port, c.Version)
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
}
+3 -3
View File
@@ -92,12 +92,12 @@ func (t *clusterTPR) findAllClusters() error {
}
func (t *clusterTPR) startCluster(c *cluster.Cluster) {
if t.devicesInUse && c.Spec.UseAllDevices {
if t.devicesInUse && c.Spec.Storage.AnyUseAllDevices() {
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace)
c.Spec.UseAllDevices = false
c.Spec.Storage.ClearUseAllDevices()
}
if c.Spec.UseAllDevices {
if c.Spec.Storage.AnyUseAllDevices() {
t.devicesInUse = true
}
+1 -1
View File
@@ -92,7 +92,7 @@ func (c *Cluster) CreateInstance() error {
}
// Start the OSDs
c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices)
c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.Storage, c.Spec.DataDirHostPath)
err = c.osds.Start()
if err != nil {
return fmt.Errorf("failed to start the osds. %+v", err)
+5 -6
View File
@@ -18,6 +18,8 @@ which also has the apache 2.0 license.
*/
package cluster
import "github.com/rook/rook/pkg/operator/osd"
type Spec struct {
// The namespace where the the rook resources will all be created.
Namespace string `json:"namespace"`
@@ -30,14 +32,11 @@ type Spec struct {
// Paused is to pause the control of the operator for the rook cluster.
Paused bool `json:"paused,omitempty"`
// Whether to consume all the storage devices found on a machine
UseAllDevices bool `json:"useAllDevices"`
// A regular expression to allow more fine-grained selection of devices on nodes across the cluster
DeviceFilter string `json:"deviceFilter"`
// The path on the host where config and data can be persisted.
DataDirHostPath string `json:"dataDirHostPath"`
// A spec for available storage in the cluster and how it should be used
Storage osd.StorageSpec `json:"storage"`
}
type PoolSpec struct {
+4
View File
@@ -46,6 +46,10 @@ func RepoPrefixEnvVar() v1.EnvVar {
return v1.EnvVar{Name: repoPrefixEnvVar, Value: repoPrefix()}
}
func ConfigDirEnvVar() v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_CONFIG_DIR", Value: DataDir}
}
func repoPrefix() string {
r := os.Getenv(repoPrefixEnvVar)
if r == "" {
@@ -13,8 +13,22 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package partition
package k8sutil
import "github.com/coreos/pkg/capnslog"
import (
"path/filepath"
"strings"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephosdpart")
func PathToVolumeName(path string) string {
// kubernetes volume names must match this regex: [a-z0-9]([-a-z0-9]*[a-z0-9])?
// first replace all filepath separators with hyphens
volumeName := strings.Replace(path, string(filepath.Separator), "-", -1)
// trim any leading/trailing hyphens
volumeName = strings.TrimPrefix(volumeName, "-")
volumeName = strings.TrimSuffix(volumeName, "-")
return volumeName
}
+1 -1
View File
@@ -150,7 +150,7 @@ func (c *Cluster) makeDeployment(id string) *extensions.Deployment {
func (c *Cluster) mdsContainer(id string) v1.Container {
command := fmt.Sprintf("/usr/bin/rookd mds --data-dir=%s --mds-id=%s ",
command := fmt.Sprintf("/usr/bin/rookd mds --config-dir=%s --mds-id=%s ",
k8sutil.DataDir, id)
return v1.Container{
// TODO: fix "sleep 5".
+1 -1
View File
@@ -86,7 +86,7 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, 1, len(cont.VolumeMounts))
assert.Equal(t, 5, len(cont.Env))
expectedCommand := fmt.Sprintf("/usr/bin/rookd mds --data-dir=/var/lib/rook --mds-id=%s ",
expectedCommand := fmt.Sprintf("/usr/bin/rookd mds --config-dir=/var/lib/rook --mds-id=%s ",
mdsID)
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
+1 -1
View File
@@ -83,7 +83,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod {
}
func (c *Cluster) monContainer(config *MonConfig, fsid string) v1.Container {
command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --port=%d --fsid=%s",
command := fmt.Sprintf("/usr/bin/rookd mon --config-dir=%s --name=%s --port=%d --fsid=%s",
k8sutil.DataDir, config.Name, config.Port, fsid)
return v1.Container{
+1 -1
View File
@@ -62,7 +62,7 @@ func testPodSpec(t *testing.T, dataDir string) {
assert.Equal(t, 1, len(cont.VolumeMounts))
assert.Equal(t, 5, len(cont.Env))
expectedCommand := fmt.Sprintf("/usr/bin/rookd mon --data-dir=/var/lib/rook --name=%s --port=%d --fsid=%s",
expectedCommand := fmt.Sprintf("/usr/bin/rookd mon --config-dir=/var/lib/rook --name=%s --port=%d --fsid=%s",
config.Name, config.Port, c.clusterInfo.FSID)
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
+190 -46
View File
@@ -18,16 +18,22 @@ package osd
import (
"fmt"
"strings"
"strconv"
"github.com/rook/rook/pkg/operator/k8sutil"
opmon "github.com/rook/rook/pkg/operator/mon"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/errors"
"k8s.io/client-go/pkg/api/unversioned"
"k8s.io/client-go/pkg/api/v1"
extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)
const (
appName = "osd"
appName = "osd"
appNameFmt = "osd-%s"
)
type Cluster struct {
@@ -35,50 +41,112 @@ type Cluster struct {
Namespace string
Keyring string
Version string
useAllDevices bool
Storage StorageSpec
dataDirHostPath string
deviceFilter string
}
func New(clientset kubernetes.Interface, namespace, version, deviceFilter, dataDirHostPath string, useAllDevices bool) *Cluster {
func New(clientset kubernetes.Interface, namespace, version string, storageSpec StorageSpec, dataDirHostPath string) *Cluster {
return &Cluster{
clientset: clientset,
Namespace: namespace,
Version: version,
deviceFilter: deviceFilter,
Storage: storageSpec,
dataDirHostPath: dataDirHostPath,
useAllDevices: useAllDevices,
}
}
func (c *Cluster) Start() error {
logger.Infof("start running osds")
logger.Infof("start running osds in namespace %s", c.Namespace)
ds, err := c.makeDaemonSet()
_, err = c.clientset.Extensions().DaemonSets(c.Namespace).Create(ds)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create osd daemon set. %+v", err)
if c.Storage.UseAllNodes {
// make a daemonset for all nodes in the cluster
ds := c.makeDaemonSet(c.Storage.Selection, c.Storage.Config)
_, err := c.clientset.Extensions().DaemonSets(c.Namespace).Create(ds)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create osd daemon set. %+v", err)
}
logger.Infof("osd daemon set already exists")
} else {
logger.Infof("osd daemon set started")
}
logger.Infof("osd daemon set already exists")
} else {
logger.Infof("osd daemon set started")
for i := range c.Storage.Nodes {
// fully resolve the storage config for this node
n := c.Storage.resolveNode(c.Storage.Nodes[i].Name)
// create the replicaSet that will run the OSDs for this node
rs := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
_, err := c.clientset.Extensions().ReplicaSets(c.Namespace).Create(rs)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create osd replica set for node %s. %+v", n.Name, err)
}
logger.Infof("osd replica set already exists for node %s", n.Name)
} else {
logger.Infof("osd replica set started for node %s", n.Name)
}
}
}
return nil
}
func (c *Cluster) makeDaemonSet() (*extensions.DaemonSet, error) {
func (c *Cluster) makeDaemonSet(selection Selection, config Config) *extensions.DaemonSet {
ds := &extensions.DaemonSet{}
ds.Name = appName
ds.Namespace = c.Namespace
podSpec := c.podTemplateSpec(nil, nil, selection, config)
ds.Spec = extensions.DaemonSetSpec{Template: podSpec}
return ds
}
func (c *Cluster) makeReplicaSet(nodeName string, devices []Device, directories []Directory,
selection Selection, config Config) *extensions.ReplicaSet {
rs := &extensions.ReplicaSet{}
rs.Name = fmt.Sprintf(appNameFmt, nodeName)
rs.Namespace = c.Namespace
podSpec := c.podTemplateSpec(devices, directories, selection, config)
podSpec.Spec.NodeSelector = map[string]string{unversioned.LabelHostname: nodeName}
replicaCount := int32(1)
rs.Spec = extensions.ReplicaSetSpec{
Template: podSpec,
Replicas: &replicaCount,
}
return rs
}
func (c *Cluster) podTemplateSpec(devices []Device, directories []Directory, selection Selection, config Config) v1.PodTemplateSpec {
// by default, the data/config dir will be an empty volume
dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}
if c.dataDirHostPath != "" {
// the user has specified a host path to use for the data dir, use that instead
dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
}
podSpec := v1.PodTemplateSpec{
// create volume config for the data dir and /dev so the pod can access devices on the host
volumes := []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource},
{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}},
}
// add each OSD directory as another host path volume source
for _, d := range directories {
dirVolume := v1.Volume{
Name: k8sutil.PathToVolumeName(d.Path),
VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: d.Path}},
}
volumes = append(volumes, dirVolume)
}
return v1.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
Name: appName,
Labels: map[string]string{
@@ -88,48 +156,124 @@ func (c *Cluster) makeDaemonSet() (*extensions.DaemonSet, error) {
Annotations: map[string]string{},
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.osdContainer()},
Containers: []v1.Container{c.osdContainer(devices, directories, selection, config)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource},
{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}},
},
Volumes: volumes,
},
}
ds.Spec = extensions.DaemonSetSpec{Template: podSpec}
return ds, nil
}
func (c *Cluster) osdContainer() v1.Container {
func (c *Cluster) osdContainer(devices []Device, directories []Directory, selection Selection, config Config) v1.Container {
command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s ", k8sutil.DataDir)
var devices string
if c.deviceFilter != "" {
devices = c.deviceFilter
} else if c.useAllDevices {
devices = "all"
command := "/usr/bin/rookd osd"
envVars := []v1.EnvVar{
opmon.ClusterNameEnvVar(),
opmon.MonEndpointEnvVar(),
opmon.MonSecretEnvVar(),
opmon.AdminSecretEnvVar(),
k8sutil.ConfigDirEnvVar(),
}
// only 1 of device list, device filter and use all devices can be specified. We prioritize in that order.
if len(devices) > 0 {
deviceNames := make([]string, len(devices))
for i := range devices {
deviceNames[i] = devices[i].Name
}
envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ",")))
} else if selection.DeviceFilter != "" {
envVars = append(envVars, deviceFilterEnvVar(selection.DeviceFilter))
} else if selection.getUseAllDevices() {
envVars = append(envVars, deviceFilterEnvVar("all"))
}
if selection.MetadataDevice != "" {
envVars = append(envVars, metadataDeviceEnvVar(selection.MetadataDevice))
}
volumeMounts := []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
{Name: "devices", MountPath: "/dev"},
}
if len(directories) > 0 {
// for each directory the user has specified, create a volume mount and pass it to the pod via cmd line arg
dirPaths := make([]string, len(directories))
for i := range directories {
dpath := directories[i].Path
dirPaths[i] = dpath
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: k8sutil.PathToVolumeName(dpath), MountPath: dpath})
}
envVars = append(envVars, dataDirectoriesEnvVar(strings.Join(dirPaths, ",")))
}
if config.StoreConfig.StoreType != "" {
envVars = append(envVars, osdStoreEnvVar(config.StoreConfig.StoreType))
}
if config.StoreConfig.DatabaseSizeMB != 0 {
envVars = append(envVars, osdDatabaseSizeEnvVar(config.StoreConfig.DatabaseSizeMB))
}
if config.StoreConfig.WalSizeMB != 0 {
envVars = append(envVars, osdWalSizeEnvVar(config.StoreConfig.WalSizeMB))
}
if config.StoreConfig.JournalSizeMB != 0 {
envVars = append(envVars, osdJournalSizeEnvVar(config.StoreConfig.JournalSizeMB))
}
if config.Location != "" {
envVars = append(envVars, locationEnvVar(config.Location))
}
privileged := true
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
{Name: "devices", MountPath: "/dev"},
},
Env: []v1.EnvVar{
v1.EnvVar{Name: "ROOKD_DATA_DEVICES", Value: devices},
opmon.ClusterNameEnvVar(),
opmon.MonEndpointEnvVar(),
opmon.MonSecretEnvVar(),
opmon.AdminSecretEnvVar(),
},
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: volumeMounts,
Env: envVars,
SecurityContext: &v1.SecurityContext{Privileged: &privileged},
}
}
func dataDevicesEnvVar(dataDevices string) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_DATA_DEVICES", Value: dataDevices}
}
func deviceFilterEnvVar(filter string) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_DATA_DEVICE_FILTER", Value: filter}
}
func metadataDeviceEnvVar(metadataDevice string) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_METADATA_DEVICE", Value: metadataDevice}
}
func dataDirectoriesEnvVar(dataDirectories string) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_DATA_DIRECTORIES", Value: dataDirectories}
}
func osdStoreEnvVar(osdStore string) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_OSD_STORE", Value: osdStore}
}
func osdDatabaseSizeEnvVar(databaseSize int) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_OSD_DATABASE_SIZE", Value: strconv.Itoa(databaseSize)}
}
func osdWalSizeEnvVar(walSize int) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_OSD_WAL_SIZE", Value: strconv.Itoa(walSize)}
}
func osdJournalSizeEnvVar(journalSize int) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_OSD_JOURNAL_SIZE", Value: strconv.Itoa(journalSize)}
}
func locationEnvVar(location string) v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_LOCATION", Value: location}
}
+130 -30
View File
@@ -16,6 +16,7 @@ limitations under the License.
package osd
import (
"strconv"
"strings"
"testing"
@@ -23,11 +24,14 @@ import (
"k8s.io/client-go/pkg/api/v1"
"github.com/stretchr/testify/assert"
"k8s.io/client-go/pkg/api/unversioned"
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
)
func TestStartDaemonset(t *testing.T) {
clientset := fake.NewSimpleClientset()
c := New(clientset, "ns", "myversion", "", "", false)
c := New(clientset, "ns", "myversion", StorageSpec{}, "")
// Start the first time
err := c.Start()
@@ -46,44 +50,140 @@ func TestDaemonset(t *testing.T) {
}
func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool) {
clientset := fake.NewSimpleClientset()
c := New(clientset, "ns", "myversion", deviceFilter, dataDir, allDevices)
daemonSet, err := c.makeDaemonSet()
assert.Nil(t, err)
assert.NotNil(t, daemonSet)
assert.Equal(t, "osd", daemonSet.Name)
assert.Equal(t, c.Namespace, daemonSet.Namespace)
assert.Equal(t, v1.RestartPolicyAlways, daemonSet.Spec.Template.Spec.RestartPolicy)
assert.Equal(t, 2, len(daemonSet.Spec.Template.Spec.Volumes))
assert.Equal(t, "rook-data", daemonSet.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, "devices", daemonSet.Spec.Template.Spec.Volumes[1].Name)
if dataDir == "" {
assert.NotNil(t, daemonSet.Spec.Template.Spec.Volumes[0].EmptyDir)
assert.Nil(t, daemonSet.Spec.Template.Spec.Volumes[0].HostPath)
} else {
assert.Nil(t, daemonSet.Spec.Template.Spec.Volumes[0].EmptyDir)
assert.Equal(t, dataDir, daemonSet.Spec.Template.Spec.Volumes[0].HostPath.Path)
storageSpec := StorageSpec{
Selection: Selection{UseAllDevices: &allDevices, DeviceFilter: deviceFilter},
Nodes: []Node{{Name: "node1"}},
}
assert.Equal(t, "osd", daemonSet.Spec.Template.ObjectMeta.Name)
assert.Equal(t, "osd", daemonSet.Spec.Template.ObjectMeta.Labels["app"])
assert.Equal(t, c.Namespace, daemonSet.Spec.Template.ObjectMeta.Labels["rook_cluster"])
assert.Equal(t, 0, len(daemonSet.Spec.Template.ObjectMeta.Annotations))
clientset := fake.NewSimpleClientset()
c := New(clientset, "ns", "myversion", storageSpec, dataDir)
cont := daemonSet.Spec.Template.Spec.Containers[0]
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
assert.NotNil(t, replicaSet)
assert.Equal(t, "osd-node1", replicaSet.Name)
assert.Equal(t, c.Namespace, replicaSet.Namespace)
assert.Equal(t, int32(1), *(replicaSet.Spec.Replicas))
assert.Equal(t, "node1", replicaSet.Spec.Template.Spec.NodeSelector[unversioned.LabelHostname])
assert.Equal(t, v1.RestartPolicyAlways, replicaSet.Spec.Template.Spec.RestartPolicy)
assert.Equal(t, 2, len(replicaSet.Spec.Template.Spec.Volumes))
assert.Equal(t, "rook-data", replicaSet.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, "devices", replicaSet.Spec.Template.Spec.Volumes[1].Name)
if dataDir == "" {
assert.NotNil(t, replicaSet.Spec.Template.Spec.Volumes[0].EmptyDir)
assert.Nil(t, replicaSet.Spec.Template.Spec.Volumes[0].HostPath)
} else {
assert.Nil(t, replicaSet.Spec.Template.Spec.Volumes[0].EmptyDir)
assert.Equal(t, dataDir, replicaSet.Spec.Template.Spec.Volumes[0].HostPath.Path)
}
assert.Equal(t, "osd", replicaSet.Spec.Template.ObjectMeta.Name)
assert.Equal(t, "osd", replicaSet.Spec.Template.ObjectMeta.Labels["app"])
assert.Equal(t, c.Namespace, replicaSet.Spec.Template.ObjectMeta.Labels["rook_cluster"])
assert.Equal(t, 0, len(replicaSet.Spec.Template.ObjectMeta.Annotations))
cont := replicaSet.Spec.Template.Spec.Containers[0]
assert.Equal(t, "quay.io/rook/rookd:myversion", cont.Image)
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, 5, len(cont.Env))
expectedCommand := "/usr/bin/rookd osd --data-dir=/var/lib/rook "
expectedCommand := "/usr/bin/rookd osd"
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
devices := cont.Env[0].Value
// verify the config dir env var
verifyEnvVar(t, cont.Env, "ROOKD_CONFIG_DIR", "/var/lib/rook", true)
// verify the osd store type env var uses the default
verifyEnvVar(t, cont.Env, "ROOKD_OSD_STORE", cephosd.DefaultStore, true)
// verify the device filter env var
if deviceFilter != "" {
assert.Equal(t, deviceFilter, devices)
verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", deviceFilter, true)
} else if allDevices {
assert.Equal(t, "all", devices)
verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", "all", true)
} else {
assert.Equal(t, "", devices)
verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", "", false)
}
}
func verifyEnvVar(t *testing.T, envVars []v1.EnvVar, expectedName, expectedValue string, expectedFound bool) {
found := false
for _, envVar := range envVars {
if envVar.Name == expectedName {
assert.Equal(t, expectedValue, envVar.Value)
found = true
break
}
}
assert.Equal(t, expectedFound, found)
}
func TestStorageSpecDevicesAndDirectories(t *testing.T) {
storageSpec := StorageSpec{
Config: Config{},
Nodes: []Node{
{
Name: "node1",
Devices: []Device{{Name: "sda"}},
Directories: []Directory{{Path: "/rook/dir1"}},
},
},
}
clientset := fake.NewSimpleClientset()
c := New(clientset, "ns", "myversion", storageSpec, "")
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
assert.NotNil(t, replicaSet)
// pod spec should have a volume for the given dir
podSpec := replicaSet.Spec.Template.Spec
assert.Equal(t, 3, len(podSpec.Volumes))
assert.Equal(t, "rook-dir1", podSpec.Volumes[2].Name)
assert.Equal(t, "/rook/dir1", podSpec.Volumes[2].VolumeSource.HostPath.Path)
// container should have a volume mount for the given dir
container := podSpec.Containers[0]
assert.Equal(t, "rook-dir1", container.VolumeMounts[2].Name)
assert.Equal(t, "/rook/dir1", container.VolumeMounts[2].MountPath)
// container command should have the given dir and device
verifyEnvVar(t, container.Env, "ROOKD_DATA_DIRECTORIES", "/rook/dir1", true)
verifyEnvVar(t, container.Env, "ROOKD_DATA_DEVICES", "sda", true)
}
func TestStorageSpecConfig(t *testing.T) {
storageSpec := StorageSpec{
Config: Config{},
Nodes: []Node{
{
Name: "node1",
Config: Config{
Location: "rack=foo",
StoreConfig: cephosd.StoreConfig{
StoreType: cephosd.Bluestore,
DatabaseSizeMB: 10,
WalSizeMB: 20,
JournalSizeMB: 30,
},
},
},
},
}
clientset := fake.NewSimpleClientset()
c := New(clientset, "ns", "myversion", storageSpec, "")
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
assert.NotNil(t, replicaSet)
container := replicaSet.Spec.Template.Spec.Containers[0]
assert.NotNil(t, container)
verifyEnvVar(t, container.Env, "ROOKD_OSD_STORE", cephosd.Bluestore, true)
verifyEnvVar(t, container.Env, "ROOKD_OSD_DATABASE_SIZE", strconv.Itoa(10), true)
verifyEnvVar(t, container.Env, "ROOKD_OSD_WAL_SIZE", strconv.Itoa(20), true)
verifyEnvVar(t, container.Env, "ROOKD_OSD_JOURNAL_SIZE", strconv.Itoa(30), true)
verifyEnvVar(t, container.Env, "ROOKD_LOCATION", "rack=foo", true)
}
+58
View File
@@ -0,0 +1,58 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package osd
import (
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
)
type StorageSpec struct {
Nodes []Node `json:"nodes,omitempty"`
UseAllNodes bool `json:"useAllNodes,omitempty"`
Selection
Config
}
type Node struct {
Name string `json:"name,omitempty"`
Devices []Device `json:"devices,omitempty"`
Directories []Directory `json:"directories,omitempty"`
Selection
Config
}
type Device struct {
Name string `json:"name,omitempty"`
}
type Directory struct {
Path string `json:"path,omitempty"`
}
type Selection struct {
// Whether to consume all the storage devices found on a machine
UseAllDevices *bool `json:"useAllDevices,omitempty"`
// A regular expression to allow more fine-grained selection of devices on nodes across the cluster
DeviceFilter string `json:"deviceFilter,omitempty"`
MetadataDevice string `json:"metadataDevice,omitempty"`
}
type Config struct {
StoreConfig cephosd.StoreConfig `json:"storeConfig,omitempty"`
Location string `json:"location,omitempty"`
}
+91
View File
@@ -0,0 +1,91 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package osd
import (
"encoding/json"
"testing"
"github.com/ghodss/yaml"
"github.com/stretchr/testify/assert"
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
)
func TestStorageSpecMarshal(t *testing.T) {
specYaml := []byte(`useAllNodes: true
useAllDevices: false
deviceFilter: "^nvme."
metadataDevice:
location: "region=us-west,datacenter=delmar"
storeConfig:
storeType: bluestore
journalSizeMB: 1024
databaseSizeMB: 1024
nodes:
- name: "node1"
storeConfig:
storeType: filestore
directories:
- path: "/rook/dir1"
- name: "node2"
deviceFilter: "^sd."`)
// convert the raw spec yaml into JSON
rawJson, err := yaml.YAMLToJSON(specYaml)
assert.Nil(t, err)
// unmarshal the JSON into a strongly typed storage spec object
var storageSpec StorageSpec
err = json.Unmarshal(rawJson, &storageSpec)
assert.Nil(t, err)
// the unmarshalled storage spec should equal the expected spec below
expectedSpec := StorageSpec{
UseAllNodes: true,
Selection: Selection{
UseAllDevices: newBool(false),
DeviceFilter: "^nvme.",
},
Config: Config{
Location: "region=us-west,datacenter=delmar",
StoreConfig: cephosd.StoreConfig{
DatabaseSizeMB: 1024,
JournalSizeMB: 1024,
StoreType: cephosd.Bluestore,
},
},
Nodes: []Node{
Node{
Name: "node1",
Directories: []Directory{{Path: "/rook/dir1"}},
Config: Config{
StoreConfig: cephosd.StoreConfig{
StoreType: cephosd.Filestore,
},
},
},
Node{
Name: "node2",
Selection: Selection{
DeviceFilter: "^sd.",
},
},
},
}
assert.Equal(t, expectedSpec, storageSpec)
}
+121
View File
@@ -0,0 +1,121 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package osd
import (
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
)
func (s *StorageSpec) AnyUseAllDevices() bool {
if s.Selection.getUseAllDevices() {
return true
}
for _, n := range s.Nodes {
if n.Selection.getUseAllDevices() {
return true
}
}
return false
}
func (s *StorageSpec) ClearUseAllDevices() {
clear := false
s.Selection.UseAllDevices = &clear
for i := range s.Nodes {
s.Nodes[i].Selection.UseAllDevices = &clear
}
}
// Fully resolves the config of the given node name, taking into account cluster level and node level specified config.
// In general, the more fine grained the configuration is specified, the more precedence it takes. Fully resolved
// configuration for the node has the following order of precedence.
// 1) Node (config defined on the node itself)
// 2) Cluster (config defined on the cluster)
// 3) Default values (if no config exists for the node or cluster)
func (s *StorageSpec) resolveNode(nodeName string) *Node {
// find the requested storage node first, if it exists
var node *Node
for i := range s.Nodes {
if s.Nodes[i].Name == nodeName {
node = &(s.Nodes[i])
break
}
}
if node == nil {
// a node with the given name was not found
return nil
}
// now resolve all properties that haven't already been set on the node
s.resolveNodeSelection(node)
s.resolveNodeConfig(node)
return node
}
func (s *StorageSpec) resolveNodeSelection(node *Node) {
resolveString(&(node.Selection.DeviceFilter), s.Selection.DeviceFilter, "")
resolveString(&(node.Selection.MetadataDevice), s.Selection.MetadataDevice, "")
if node.Selection.UseAllDevices == nil {
if s.Selection.UseAllDevices != nil {
// the node does not have a value specified for use all devices, but the cluster does. Use the cluster's.
node.Selection.UseAllDevices = s.Selection.UseAllDevices
} else {
// neither node nor cluster have a value set for use all devices, use the default value.
node.Selection.UseAllDevices = newBool(false)
}
}
}
func (s *StorageSpec) resolveNodeConfig(node *Node) {
resolveString(&(node.Config.StoreConfig.StoreType), s.Config.StoreConfig.StoreType, cephosd.DefaultStore)
resolveInt(&(node.Config.StoreConfig.DatabaseSizeMB), s.Config.StoreConfig.DatabaseSizeMB, 0)
resolveInt(&(node.Config.StoreConfig.WalSizeMB), s.Config.StoreConfig.WalSizeMB, 0)
resolveInt(&(node.Config.StoreConfig.JournalSizeMB), s.Config.StoreConfig.JournalSizeMB, 0)
resolveString(&(node.Config.Location), s.Config.Location, "")
}
func (s *Selection) getUseAllDevices() bool {
return s.UseAllDevices != nil && *(s.UseAllDevices)
}
func resolveString(setting *string, parent, defaultVal string) {
if *setting == "" {
if parent != "" {
*setting = parent
} else {
*setting = defaultVal
}
}
}
func resolveInt(setting *int, parent, defaultVal int) {
if *setting == 0 {
if parent != 0 {
*setting = parent
} else {
*setting = defaultVal
}
}
}
func newBool(val bool) *bool {
return &val
}
+169
View File
@@ -0,0 +1,169 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package osd
import (
"testing"
"github.com/stretchr/testify/assert"
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
)
func TestResolveNodeNotExist(t *testing.T) {
// a non existing node should return nil
storageSpec := StorageSpec{}
node := storageSpec.resolveNode("fake node")
assert.Nil(t, node)
// a node with no properties defined should inherit them from the cluster storage spec
storageSpec = StorageSpec{
Selection: Selection{
DeviceFilter: "^sd.",
MetadataDevice: "nvme01",
},
Config: Config{
Location: "root=default,row=a,rack=a2,chassis=a2a,host=a2a1",
StoreConfig: cephosd.StoreConfig{
StoreType: cephosd.Bluestore,
DatabaseSizeMB: 1024,
WalSizeMB: 128,
JournalSizeMB: 2048,
},
},
Nodes: []Node{
{Name: "node1"}, // node gets nothing but its name set
},
}
node = storageSpec.resolveNode("node1")
assert.NotNil(t, node)
assert.Equal(t, "^sd.", node.Selection.DeviceFilter)
assert.Equal(t, "nvme01", node.Selection.MetadataDevice)
assert.False(t, node.Selection.getUseAllDevices())
assert.Equal(t, "root=default,row=a,rack=a2,chassis=a2a,host=a2a1", node.Config.Location)
assert.Equal(t, cephosd.Bluestore, node.Config.StoreConfig.StoreType)
assert.Equal(t, 1024, node.Config.StoreConfig.DatabaseSizeMB)
assert.Equal(t, 128, node.Config.StoreConfig.WalSizeMB)
assert.Equal(t, 2048, node.Config.StoreConfig.JournalSizeMB)
}
func TestResolveNodeInherentFromCluster(t *testing.T) {
// a node with no properties defined should inherit them from the cluster storage spec
storageSpec := StorageSpec{
Selection: Selection{
DeviceFilter: "^sd.",
MetadataDevice: "nvme01",
},
Config: Config{
Location: "root=default,row=a,rack=a2,chassis=a2a,host=a2a1",
StoreConfig: cephosd.StoreConfig{
StoreType: cephosd.Bluestore,
DatabaseSizeMB: 1024,
WalSizeMB: 128,
JournalSizeMB: 2048,
},
},
Nodes: []Node{
{Name: "node1"}, // node gets nothing but its name set
},
}
node := storageSpec.resolveNode("node1")
assert.NotNil(t, node)
assert.Equal(t, "^sd.", node.Selection.DeviceFilter)
assert.Equal(t, "nvme01", node.Selection.MetadataDevice)
assert.False(t, node.Selection.getUseAllDevices())
assert.Equal(t, "root=default,row=a,rack=a2,chassis=a2a,host=a2a1", node.Config.Location)
assert.Equal(t, cephosd.Bluestore, node.Config.StoreConfig.StoreType)
assert.Equal(t, 1024, node.Config.StoreConfig.DatabaseSizeMB)
assert.Equal(t, 128, node.Config.StoreConfig.WalSizeMB)
assert.Equal(t, 2048, node.Config.StoreConfig.JournalSizeMB)
}
func TestResolveNodeDefaultValues(t *testing.T) {
// a node with no properties and none defined in the cluster storage spec should get the default values
storageSpec := StorageSpec{
Nodes: []Node{
{Name: "node1"}, // node gets nothing but its name set
},
}
node := storageSpec.resolveNode("node1")
assert.NotNil(t, node)
assert.Equal(t, "", node.Selection.DeviceFilter)
assert.Equal(t, "", node.Selection.MetadataDevice)
assert.False(t, node.Selection.getUseAllDevices())
assert.Equal(t, "", node.Config.Location)
assert.Equal(t, cephosd.Filestore, node.Config.StoreConfig.StoreType)
assert.Equal(t, 0, node.Config.StoreConfig.DatabaseSizeMB)
assert.Equal(t, 0, node.Config.StoreConfig.WalSizeMB)
assert.Equal(t, 0, node.Config.StoreConfig.JournalSizeMB)
}
func TestResolveNodeUseAllDevices(t *testing.T) {
storageSpec := StorageSpec{
Selection: Selection{UseAllDevices: newBool(true)}, // UseAllDevices is set to true on the storage spec
Nodes: []Node{
{Name: "node1"}, // node gets nothing but its name set
},
}
node := storageSpec.resolveNode("node1")
assert.NotNil(t, node)
assert.True(t, node.Selection.getUseAllDevices())
}
func TestUseAllDevices(t *testing.T) {
storageSpec := StorageSpec{}
assert.False(t, storageSpec.AnyUseAllDevices())
storageSpec = StorageSpec{
Selection: Selection{
UseAllDevices: newBool(true)}, // UseAllDevices is set to true on the storage spec
}
assert.True(t, storageSpec.AnyUseAllDevices())
storageSpec = StorageSpec{
Selection: Selection{UseAllDevices: newBool(false)},
Nodes: []Node{
{
Name: "node1",
Selection: Selection{UseAllDevices: newBool(true)},
},
},
}
assert.True(t, storageSpec.AnyUseAllDevices())
}
func TestClearUseAllDevices(t *testing.T) {
// create a storage spec with use all devices set to true for the cluster and for all nodes
storageSpec := StorageSpec{
Selection: Selection{UseAllDevices: newBool(true)},
Nodes: []Node{
{
Name: "node1",
Selection: Selection{UseAllDevices: newBool(true)},
},
},
}
assert.True(t, storageSpec.AnyUseAllDevices())
// now clear the use all devices field, it should be cleared from the entire cluster and its nodes
storageSpec.ClearUseAllDevices()
assert.False(t, storageSpec.AnyUseAllDevices())
}
+1 -1
View File
@@ -158,7 +158,7 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
func (c *Cluster) rgwContainer() v1.Container {
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --rgw-port=%d --rgw-host=%s",
command := fmt.Sprintf("/usr/bin/rookd rgw --config-dir=%s --rgw-port=%d --rgw-host=%s",
k8sutil.DataDir, cephrgw.RGWPort, cephrgw.DNSName)
return v1.Container{
// TODO: fix "sleep 5".
+1 -1
View File
@@ -95,7 +95,7 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, 1, len(cont.VolumeMounts))
assert.Equal(t, 5, len(cont.Env))
expectedCommand := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=/var/lib/rook --rgw-port=%d --rgw-host=%s",
expectedCommand := fmt.Sprintf("/usr/bin/rookd rgw --config-dir=/var/lib/rook --rgw-port=%d --rgw-host=%s",
cephrgw.RGWPort, cephrgw.DNSName)
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
+6
View File
@@ -18,6 +18,7 @@ package exec
import (
"bufio"
"io"
"os"
"os/exec"
"strings"
@@ -30,6 +31,7 @@ type Executor interface {
ExecuteCommandPipeline(actionName string, command string) (string, error)
ExecuteCommandWithOutput(actionName string, command string, arg ...string) (string, error)
ExecuteCommandWithCombinedOutput(actionName string, command string, arg ...string) (string, error)
ExecuteStat(name string) (os.FileInfo, error)
}
type CommandExecutor struct {
@@ -93,6 +95,10 @@ func startCommand(command string, arg ...string) (*exec.Cmd, io.ReadCloser, io.R
return cmd, stdout, stderr, err
}
func (*CommandExecutor) ExecuteStat(name string) (os.FileInfo, error) {
return os.Stat(name)
}
func logOutput(name string, stdout, stderr io.ReadCloser) {
if stdout == nil || stderr == nil {
logger.Warningf("failed to collect stdout and stderr")
+13 -1
View File
@@ -15,7 +15,10 @@ limitations under the License.
*/
package test
import "os/exec"
import (
"os"
"os/exec"
)
// ******************** MockExecutor ********************
type MockExecutor struct {
@@ -24,6 +27,7 @@ type MockExecutor struct {
MockExecuteCommandPipeline func(actionName string, command string) (string, error)
MockExecuteCommandWithOutput func(actionName string, command string, arg ...string) (string, error)
MockExecuteCommandWithCombinedOutput func(actionName string, command string, arg ...string) (string, error)
MockExecuteStat func(name string) (os.FileInfo, error)
}
func (e *MockExecutor) ExecuteCommand(actionName string, command string, arg ...string) error {
@@ -66,3 +70,11 @@ func (e *MockExecutor) ExecuteCommandWithCombinedOutput(actionName string, comma
return "", nil
}
func (e *MockExecutor) ExecuteStat(name string) (os.FileInfo, error) {
if e.MockExecuteStat != nil {
return e.MockExecuteStat(name)
}
return nil, nil
}
+4 -4
View File
@@ -146,7 +146,7 @@ func CreatePartitions(device string, args []string, executor exec.Executor) erro
func FormatDevice(devicePath string, executor exec.Executor) error {
cmd := fmt.Sprintf("mkfs.ext4 %s", devicePath)
if err := executor.ExecuteCommand(cmd, "sudo", "mkfs.ext4", devicePath); err != nil {
if err := executor.ExecuteCommand(cmd, "mkfs.ext4", devicePath); err != nil {
return fmt.Errorf("command %s failed: %+v", cmd, err)
}
@@ -197,7 +197,7 @@ func MountDevice(devicePath, mountPath string, executor exec.Executor) error {
// comma-separated list of mount options passed directly to mount command
func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, executor exec.Executor) error {
args := []string{"mount"}
args := []string{}
if fstype != "" {
args = append(args, "-t", fstype)
@@ -212,7 +212,7 @@ func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, execu
os.MkdirAll(mountPath, 0755)
cmd := fmt.Sprintf("mount %s", devicePath)
if err := executor.ExecuteCommand(cmd, "sudo", args...); err != nil {
if err := executor.ExecuteCommand(cmd, "mount", args...); err != nil {
return fmt.Errorf("command %s failed: %+v", cmd, err)
}
@@ -221,7 +221,7 @@ func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, execu
func UnmountDevice(devicePath string, executor exec.Executor) error {
cmd := fmt.Sprintf("umount %s", devicePath)
if err := executor.ExecuteCommand(cmd, "sudo", "umount", devicePath); err != nil {
if err := executor.ExecuteCommand(cmd, "umount", devicePath); err != nil {
cmdErr, ok := err.(*exec.CommandError)
if ok && cmdErr.ExitStatus() == 32 {
logger.Infof("ignoring exit status 32 from unmount of device %s, err:%+v", devicePath, cmdErr)
+4 -4
View File
@@ -94,13 +94,13 @@ func TestMountDeviceWithOptions(t *testing.T) {
MockExecuteCommand: func(actionName string, command string, arg ...string) error {
switch testCount {
case 0:
assert.Equal(t, []string{"mount", "/dev/abc1", "/tmp/mount1"}, arg)
assert.Equal(t, []string{"/dev/abc1", "/tmp/mount1"}, arg)
case 1:
assert.Equal(t, []string{"mount", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg)
assert.Equal(t, []string{"-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg)
case 2:
assert.Equal(t, []string{"mount", "-t", "myfstype", "/dev/abc1", "/tmp/mount1"}, arg)
assert.Equal(t, []string{"-t", "myfstype", "/dev/abc1", "/tmp/mount1"}, arg)
case 3:
assert.Equal(t, []string{"mount", "-t", "myfstype", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg)
assert.Equal(t, []string{"-t", "myfstype", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg)
}
testCount++