Merge pull request #529 from jbw976/storage-config
Kubernetes operator and cephmgr support for specifying storage resour…
This commit is contained in:
commit
c2d6f34fa8
45 files changed
+1558
-476
No files matched your search
@@ -17,7 +17,7 @@ Rook can be deployed as a standalone service directly on any modern Linux host b
|
||||
|
||||
2. Start a one node Rook cluster
|
||||
```bash
|
||||
./rookd --data-dir /tmp/rook-test
|
||||
./rookd --config-dir /tmp/rook-test
|
||||
```
|
||||
|
||||
3. In another console (but in the same path), verify the Rook cluster is ready
|
||||
|
||||
@@ -15,7 +15,7 @@ build() {
|
||||
trap "rm -fr $tmpdir" EXIT
|
||||
cat <<EOF > $tmpdir/Dockerfile
|
||||
FROM alpine:3.5
|
||||
RUN apk add --no-cache gptfdisk util-linux coreutils
|
||||
RUN apk add --no-cache gptfdisk util-linux coreutils e2fsprogs
|
||||
COPY root /
|
||||
ENTRYPOINT ["/usr/bin/rookd"]
|
||||
EOF
|
||||
|
||||
@@ -38,7 +38,8 @@ func TestMountFilesystem(t *testing.T) {
|
||||
}
|
||||
e := &exectest.MockExecutor{
|
||||
MockExecuteCommand: func(actionName string, command string, arg ...string) error {
|
||||
expectedArgs := []string{"mount", "-t", "ceph", "-o", "name=admin,secret=AQBsCv1X5oD9GhAARHVU9N+kFRWDjyLA1dqzIg==",
|
||||
assert.Equal(t, "mount", command)
|
||||
expectedArgs := []string{"-t", "ceph", "-o", "name=admin,secret=AQBsCv1X5oD9GhAARHVU9N+kFRWDjyLA1dqzIg==",
|
||||
"10.37.129.214:6790:/", "/tmp/myfs1mount"}
|
||||
assert.Equal(t, expectedArgs, arg)
|
||||
return nil
|
||||
|
||||
@@ -27,7 +27,8 @@ import (
|
||||
func TestUnmountFilesystem(t *testing.T) {
|
||||
e := &exectest.MockExecutor{
|
||||
MockExecuteCommand: func(actionName string, command string, arg ...string) error {
|
||||
expectedArgs := []string{"umount", "/tmp/myfs1mount"}
|
||||
assert.Equal(t, "umount", command)
|
||||
expectedArgs := []string{"/tmp/myfs1mount"}
|
||||
assert.Equal(t, expectedArgs, arg)
|
||||
return nil
|
||||
},
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@ func init() {
|
||||
}
|
||||
|
||||
func startAPI(cmd *cobra.Command, args []string) error {
|
||||
if err := flags.VerifyRequiredFlags(apiCmd, []string{"repo-prefix", "namespace", "data-dir", "cluster-name", "mon-endpoints", "mon-secret", "admin-secret"}); err != nil {
|
||||
if err := flags.VerifyRequiredFlags(apiCmd, []string{"repo-prefix", "namespace", "config-dir", "cluster-name", "mon-endpoints", "mon-secret", "admin-secret"}); err != nil {
|
||||
return err
|
||||
}
|
||||
if apiPort == 0 {
|
||||
|
||||
+5
-3
@@ -33,7 +33,7 @@ import (
|
||||
"github.com/rook/rook/pkg/cephmgr"
|
||||
"github.com/rook/rook/pkg/cephmgr/cephd"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd/partition"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/model"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
@@ -62,6 +62,7 @@ type config struct {
|
||||
discoveryURL string
|
||||
etcdMembers string
|
||||
devices string
|
||||
directories string
|
||||
metadataDevice string
|
||||
dataDir string
|
||||
adminSecret string
|
||||
@@ -69,7 +70,7 @@ type config struct {
|
||||
location string
|
||||
logLevel capnslog.LogLevel
|
||||
cephConfigOverride string
|
||||
bluestoreConfig partition.BluestoreConfig
|
||||
storeConfig osd.StoreConfig
|
||||
networkInfo clusterd.NetworkInfo
|
||||
monEndpoints string
|
||||
namespace string
|
||||
@@ -160,7 +161,8 @@ func joinCluster() error {
|
||||
}
|
||||
|
||||
services := []*clusterd.ClusterService{
|
||||
cephmgr.NewCephService(cephd.New(), cfg.devices, cfg.metadataDevice, cfg.forceFormat, cfg.location, cfg.adminSecret, cfg.bluestoreConfig),
|
||||
cephmgr.NewCephService(cephd.New(), cfg.devices, cfg.metadataDevice, cfg.directories,
|
||||
cfg.forceFormat, cfg.location, cfg.adminSecret, cfg.storeConfig),
|
||||
}
|
||||
|
||||
cfg.nodeID, err = util.LoadPersistedNodeID(cfg.dataDir)
|
||||
|
||||
+2
-2
@@ -46,7 +46,7 @@ func addCephFlags(command *cobra.Command) {
|
||||
command.Flags().StringVar(&clusterInfo.MonitorSecret, "mon-secret", "", "the cephx keyring for monitors")
|
||||
command.Flags().StringVar(&clusterInfo.AdminSecret, "admin-secret", "", "secret for the admin user (random if not specified)")
|
||||
command.Flags().StringVar(&cfg.monEndpoints, "mon-endpoints", "", "ceph mon endpoints")
|
||||
command.Flags().StringVar(&cfg.dataDir, "data-dir", "/var/lib/rook", "directory for storing configuration")
|
||||
command.Flags().StringVar(&cfg.dataDir, "config-dir", "/var/lib/rook", "directory for storing configuration")
|
||||
command.Flags().StringVar(&cfg.cephConfigOverride, "ceph-config-override", "", "optional path to a ceph config file that will be appended to the config files that rook generates")
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ func init() {
|
||||
}
|
||||
|
||||
func startMon(cmd *cobra.Command, args []string) error {
|
||||
if err := flags.VerifyRequiredFlags(monCmd, []string{"name", "fsid", "mon-secret", "admin-secret", "data-dir", "cluster-name", "private-ipv4"}); err != nil {
|
||||
if err := flags.VerifyRequiredFlags(monCmd, []string{"name", "fsid", "mon-secret", "admin-secret", "config-dir", "cluster-name", "private-ipv4"}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+29
-10
@@ -18,10 +18,11 @@ limitations under the License.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/cephd"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd/partition"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/util/flags"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -33,18 +34,24 @@ var osdCmd = &cobra.Command{
|
||||
Hidden: true,
|
||||
}
|
||||
var (
|
||||
osdCluster mon.ClusterInfo
|
||||
osdCluster mon.ClusterInfo
|
||||
osdDataDeviceFilter string
|
||||
)
|
||||
|
||||
func addOSDFlags(command *cobra.Command) {
|
||||
command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement")
|
||||
command.Flags().StringVar(&cfg.devices, "data-devices", "", "comma separated list of devices to use for storage, or \"all\"")
|
||||
command.Flags().StringVar(&cfg.devices, "data-devices", "", "comma separated list of devices to use for storage")
|
||||
command.Flags().StringVar(&osdDataDeviceFilter, "data-device-filter", "", "a regex filter for the device names to use, or \"all\"")
|
||||
command.Flags().StringVar(&cfg.directories, "data-directories", "", "comma separated list of directory paths to use for storage")
|
||||
command.Flags().StringVar(&cfg.metadataDevice, "metadata-device", "", "device to use for metadata (e.g. a high performance SSD/NVMe device)")
|
||||
command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement")
|
||||
command.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
|
||||
"true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!")
|
||||
// bluestore config flags
|
||||
command.Flags().IntVar(&cfg.bluestoreConfig.WalSizeMB, "osd-wal-size", partition.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL)")
|
||||
command.Flags().IntVar(&cfg.bluestoreConfig.DatabaseSizeMB, "osd-database-size", partition.DBDefaultSizeMB, "default size (MB) for OSD database")
|
||||
|
||||
// OSD store config flags
|
||||
command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osd.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)")
|
||||
command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", osd.DBDefaultSizeMB, "default size (MB) for OSD database (bluestore)")
|
||||
command.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osd.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)")
|
||||
command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", osd.Filestore, "type of backing OSD store to use (bluestore or filestore)")
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -60,13 +67,25 @@ func startOSD(cmd *cobra.Command, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
var dataDevices string
|
||||
var usingDeviceFilter bool
|
||||
if osdDataDeviceFilter != "" {
|
||||
if cfg.devices != "" {
|
||||
return fmt.Errorf("Only one of --data-devices and --data-device-filter can be specified.")
|
||||
}
|
||||
|
||||
dataDevices = osdDataDeviceFilter
|
||||
usingDeviceFilter = true
|
||||
} else {
|
||||
dataDevices = cfg.devices
|
||||
}
|
||||
|
||||
setLogLevel()
|
||||
|
||||
metadataDevice := ""
|
||||
forceFormat := false
|
||||
location := ""
|
||||
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
|
||||
agent := osd.NewAgent(cephd.New(), cfg.devices, metadataDevice, forceFormat, location, cfg.bluestoreConfig, &clusterInfo)
|
||||
agent := osd.NewAgent(cephd.New(), dataDevices, usingDeviceFilter, cfg.metadataDevice, cfg.directories, forceFormat,
|
||||
cfg.location, cfg.storeConfig, &clusterInfo)
|
||||
context := clusterd.NewDaemonContext(cfg.dataDir, cfg.cephConfigOverride, cfg.logLevel)
|
||||
|
||||
return osd.Run(context, agent)
|
||||
|
||||
Generated
+21
-8
@@ -1,5 +1,5 @@
|
||||
hash: 81178d6bc6321564bed3985f0746ccd7357067d5c9eaecfdccdbd86cfd897387
|
||||
updated: 2017-03-08T10:56:46.619273095-08:00
|
||||
hash: 23db89af71e3e11cfa7bb7722f8dd338f3285dd87270f8e6000898662e7def08
|
||||
updated: 2017-04-03T13:42:53.811824499-07:00
|
||||
imports:
|
||||
- name: bitbucket.org/ww/goautoneg
|
||||
version: a547fc61f48d567d5b4ec6f8aee5573d8efce11d
|
||||
@@ -150,7 +150,7 @@ imports:
|
||||
- name: github.com/inconshreveable/mousetrap
|
||||
version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
|
||||
- name: github.com/jonboulle/clockwork
|
||||
version: 72f9bd7c4e0c2a40055ab3d0f09654f730cce982
|
||||
version: 2eee05ed794112d45db504eb05aa693efd2b8b09
|
||||
- name: github.com/juju/ratelimit
|
||||
version: 77ed1c8a01217656d2080ad51981f6e99adaa177
|
||||
- name: github.com/mailru/easyjson
|
||||
@@ -198,13 +198,12 @@ imports:
|
||||
- name: github.com/xiang90/probing
|
||||
version: 07dd2e8dfe18522e9c447ba95f2fe95262f63bb2
|
||||
- name: golang.org/x/crypto
|
||||
version: 1f22c0103821b9390939b6776727195525381532
|
||||
version: 1351f936d976c60a0a48d728281922cf63eafb8d
|
||||
subpackages:
|
||||
- bcrypt
|
||||
- blowfish
|
||||
- ssh/terminal
|
||||
- name: golang.org/x/net
|
||||
version: e90d6d0afc4c315a0d87a568ae68577cc15149a0
|
||||
version: 6acef71eb69611914f7a30939ea9f6e194c78172
|
||||
subpackages:
|
||||
- context
|
||||
- context/ctxhttp
|
||||
@@ -212,7 +211,6 @@ imports:
|
||||
- http2/hpack
|
||||
- idna
|
||||
- internal/timeseries
|
||||
- lex/httplex
|
||||
- trace
|
||||
- name: golang.org/x/oauth2
|
||||
version: 3c3a985cb79f52a3190fbc056984415ca6763d01
|
||||
@@ -222,7 +220,7 @@ imports:
|
||||
- jws
|
||||
- jwt
|
||||
- name: golang.org/x/sys
|
||||
version: 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9
|
||||
version: 9c60d1c508f5134d1ca726b4641db998f2523357
|
||||
subpackages:
|
||||
- unix
|
||||
- name: golang.org/x/text
|
||||
@@ -269,19 +267,33 @@ imports:
|
||||
version: e121606b0d09b2e1c467183ee46217fa85a6b672
|
||||
subpackages:
|
||||
- discovery
|
||||
- discovery/fake
|
||||
- kubernetes
|
||||
- kubernetes/fake
|
||||
- kubernetes/typed/apps/v1beta1
|
||||
- kubernetes/typed/apps/v1beta1/fake
|
||||
- kubernetes/typed/authentication/v1beta1
|
||||
- kubernetes/typed/authentication/v1beta1/fake
|
||||
- kubernetes/typed/authorization/v1beta1
|
||||
- kubernetes/typed/authorization/v1beta1/fake
|
||||
- kubernetes/typed/autoscaling/v1
|
||||
- kubernetes/typed/autoscaling/v1/fake
|
||||
- kubernetes/typed/batch/v1
|
||||
- kubernetes/typed/batch/v1/fake
|
||||
- kubernetes/typed/batch/v2alpha1
|
||||
- kubernetes/typed/batch/v2alpha1/fake
|
||||
- kubernetes/typed/certificates/v1alpha1
|
||||
- kubernetes/typed/certificates/v1alpha1/fake
|
||||
- kubernetes/typed/core/v1
|
||||
- kubernetes/typed/core/v1/fake
|
||||
- kubernetes/typed/extensions/v1beta1
|
||||
- kubernetes/typed/extensions/v1beta1/fake
|
||||
- kubernetes/typed/policy/v1beta1
|
||||
- kubernetes/typed/policy/v1beta1/fake
|
||||
- kubernetes/typed/rbac/v1alpha1
|
||||
- kubernetes/typed/rbac/v1alpha1/fake
|
||||
- kubernetes/typed/storage/v1beta1
|
||||
- kubernetes/typed/storage/v1beta1/fake
|
||||
- pkg/api
|
||||
- pkg/api/errors
|
||||
- pkg/api/install
|
||||
@@ -370,6 +382,7 @@ imports:
|
||||
- plugin/pkg/client/auth/gcp
|
||||
- plugin/pkg/client/auth/oidc
|
||||
- rest
|
||||
- testing
|
||||
- tools/clientcmd/api
|
||||
- tools/metrics
|
||||
- transport
|
||||
|
||||
@@ -19,6 +19,7 @@ import:
|
||||
- package: github.com/spf13/pflag
|
||||
- package: github.com/google/uuid
|
||||
- package: github.com/go-ini/ini
|
||||
- package: github.com/ghodss/yaml
|
||||
- package: github.com/gorilla/mux
|
||||
- package: bitbucket.org/ww/goautoneg
|
||||
version: a547fc61f48d567d5b4ec6f8aee5573d8efce11d
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/rook/rook/pkg/cephmgr/mds"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd/partition"
|
||||
"github.com/rook/rook/pkg/cephmgr/rgw"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
)
|
||||
@@ -30,15 +29,15 @@ const (
|
||||
)
|
||||
|
||||
// create a new ceph service
|
||||
func NewCephService(factory client.ConnectionFactory, devices, metadataDevice string, forceFormat bool,
|
||||
location, adminSecret string, bluestoreConfig partition.BluestoreConfig) *clusterd.ClusterService {
|
||||
func NewCephService(factory client.ConnectionFactory, devices, metadataDevice, directories string, forceFormat bool,
|
||||
location, adminSecret string, storeConfig osd.StoreConfig) *clusterd.ClusterService {
|
||||
|
||||
return &clusterd.ClusterService{
|
||||
Name: cephName,
|
||||
Leader: newLeader(factory, adminSecret),
|
||||
Agents: []clusterd.ServiceAgent{
|
||||
mon.NewAgent(factory),
|
||||
osd.NewAgent(factory, devices, metadataDevice, forceFormat, location, bluestoreConfig, nil),
|
||||
osd.NewAgent(factory, devices, false, metadataDevice, directories, forceFormat, location, storeConfig, nil),
|
||||
mds.NewAgent(factory),
|
||||
rgw.NewAgent(factory),
|
||||
},
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
|
||||
testceph "github.com/rook/rook/pkg/cephmgr/client/test"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd/partition"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/clusterd/inventory"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
@@ -138,7 +138,7 @@ func TestRefreshKeys(t *testing.T) {
|
||||
func TestNewCephService(t *testing.T) {
|
||||
factory := &testceph.MockConnectionFactory{Fsid: "fsid", SecretKey: "key"}
|
||||
|
||||
service := NewCephService(factory, "a,b,c", "", true, "root=default", "", partition.BluestoreConfig{})
|
||||
service := NewCephService(factory, "a,b,c", "", "", true, "root=default", "", osd.StoreConfig{})
|
||||
assert.NotNil(t, service)
|
||||
assert.Equal(t, "/rook/services/ceph/osd/desired", service.Leader.RefreshKeys()[0].Path)
|
||||
assert.Equal(t, 4, len(service.Agents))
|
||||
|
||||
+94
-57
@@ -35,7 +35,6 @@ import (
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd/partition"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
"github.com/rook/rook/pkg/util/proc"
|
||||
@@ -53,24 +52,27 @@ const (
|
||||
)
|
||||
|
||||
type OsdAgent struct {
|
||||
cluster *mon.ClusterInfo
|
||||
forceFormat bool
|
||||
location string
|
||||
factory client.ConnectionFactory
|
||||
osdProc map[int]*proc.MonitoredProc
|
||||
desiredDevices []string
|
||||
devices string
|
||||
metadataDevice string
|
||||
bluestoreConfig partition.BluestoreConfig
|
||||
configCounter int32
|
||||
osdsCompleted chan struct{}
|
||||
cluster *mon.ClusterInfo
|
||||
forceFormat bool
|
||||
location string
|
||||
factory client.ConnectionFactory
|
||||
osdProc map[int]*proc.MonitoredProc
|
||||
desiredDevices []string
|
||||
desiredDirectories []string
|
||||
devices string
|
||||
usingDeviceFilter bool
|
||||
metadataDevice string
|
||||
directories string
|
||||
storeConfig StoreConfig
|
||||
configCounter int32
|
||||
osdsCompleted chan struct{}
|
||||
}
|
||||
|
||||
func NewAgent(factory client.ConnectionFactory, devices, metadataDevice string, forceFormat bool,
|
||||
location string, bluestoreConfig partition.BluestoreConfig, cluster *mon.ClusterInfo) *OsdAgent {
|
||||
func NewAgent(factory client.ConnectionFactory, devices string, usingDeviceFilter bool, metadataDevice, directories string, forceFormat bool,
|
||||
location string, storeConfig StoreConfig, cluster *mon.ClusterInfo) *OsdAgent {
|
||||
|
||||
return &OsdAgent{factory: factory, devices: devices, metadataDevice: metadataDevice, forceFormat: forceFormat,
|
||||
location: location, bluestoreConfig: bluestoreConfig, cluster: cluster}
|
||||
return &OsdAgent{factory: factory, devices: devices, usingDeviceFilter: usingDeviceFilter, metadataDevice: metadataDevice,
|
||||
directories: directories, forceFormat: forceFormat, location: location, storeConfig: storeConfig, cluster: cluster}
|
||||
}
|
||||
|
||||
func (a *OsdAgent) Name() string {
|
||||
@@ -86,13 +88,24 @@ func (a *OsdAgent) Initialize(context *clusterd.Context) error {
|
||||
logger.Infof("desired devices for osds: %+v. desired metadata device: %s", a.desiredDevices, a.metadataDevice)
|
||||
}
|
||||
|
||||
if len(a.directories) > 0 {
|
||||
a.desiredDirectories = strings.Split(a.directories, ",")
|
||||
}
|
||||
|
||||
// if no devices or directories were specified, use the current directory for an osd
|
||||
if len(a.desiredDevices) == 0 {
|
||||
if len(a.desiredDevices) == 0 && len(a.desiredDirectories) == 0 {
|
||||
logger.Infof("Adding local path %s to desired state", context.ConfigDir)
|
||||
err := AddDesiredDir(context.EtcdClient, context.ConfigDir, context.NodeID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add current dir %s. %v", context.ConfigDir, err)
|
||||
a.desiredDirectories = []string{context.ConfigDir}
|
||||
}
|
||||
|
||||
if len(a.desiredDirectories) > 0 {
|
||||
for _, dir := range a.desiredDirectories {
|
||||
err := AddDesiredDir(context.EtcdClient, dir, context.NodeID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add desired dir %s. %v", dir, err)
|
||||
}
|
||||
}
|
||||
logger.Infof("desired directories for osds: %+v.", a.desiredDirectories)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -351,7 +364,7 @@ func (a *OsdAgent) configureDirs(context *clusterd.Context, dirs map[string]int)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("%d/%d osds (filestore) succeeded on this node", succeeded, len(dirs))
|
||||
logger.Infof("%d/%d osd dirs succeeded on this node", succeeded, len(dirs))
|
||||
return lastErr
|
||||
|
||||
}
|
||||
@@ -387,7 +400,7 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
|
||||
defer a.decrementConfigCounter()
|
||||
|
||||
// compute an OSD layout scheme that will optimize performance
|
||||
scheme, err := getPartitionPerfScheme(context, connection, devices, a.bluestoreConfig)
|
||||
scheme, err := getPartitionPerfScheme(context, connection, devices, a.storeConfig)
|
||||
logger.Debugf("partition scheme: %+v, err: %+v", scheme, err)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get OSD performance scheme: %+v", err)
|
||||
@@ -396,8 +409,8 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
|
||||
|
||||
if scheme.Metadata != nil {
|
||||
// partition the dedicated metadata device
|
||||
if err := partitionBluestoreMetadata(context, scheme.Metadata, context.ConfigDir); err != nil {
|
||||
logger.Errorf("failed to partition bluestore metadata %+v: %+v", scheme.Metadata, err)
|
||||
if err := partitionMetadata(context, scheme.Metadata, context.ConfigDir); err != nil {
|
||||
logger.Errorf("failed to partition metadata %+v: %+v", scheme.Metadata, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -414,7 +427,7 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("%d/%d bluestore osds succeeded on this node", succeeded, len(scheme.Entries))
|
||||
logger.Infof("%d/%d osd devices succeeded on this node", succeeded, len(scheme.Entries))
|
||||
}()
|
||||
|
||||
return nil
|
||||
@@ -423,10 +436,10 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
|
||||
// computes a partitioning scheme for all the given desired devices. This could be devics already in use,
|
||||
// devices dedicated to metadata, and devices with all bluestore partitions collocated.
|
||||
func getPartitionPerfScheme(context *clusterd.Context, connection client.Connection, devices *DeviceOsdMapping,
|
||||
bluestoreConfig partition.BluestoreConfig) (*partition.PerfScheme, error) {
|
||||
storeConfig StoreConfig) (*PerfScheme, error) {
|
||||
|
||||
// load the existing (committed) partition scheme from disk
|
||||
perfScheme, err := partition.LoadScheme(context.ConfigDir)
|
||||
perfScheme, err := LoadScheme(context.ConfigDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load partition scheme from %s: %+v", context.ConfigDir, err)
|
||||
}
|
||||
@@ -461,7 +474,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
|
||||
}
|
||||
|
||||
metadataEntry = mapping
|
||||
perfScheme.Metadata = partition.NewMetadataDeviceInfo(name)
|
||||
perfScheme.Metadata = NewMetadataDeviceInfo(name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -478,7 +491,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
|
||||
return nil, fmt.Errorf("failed to register OSD for device %s: %+v", name, err)
|
||||
}
|
||||
|
||||
schemeEntry := partition.NewPerfSchemeEntry()
|
||||
schemeEntry := NewPerfSchemeEntry(storeConfig.StoreType)
|
||||
schemeEntry.ID = *osdID
|
||||
schemeEntry.OsdUUID = *osdUUID
|
||||
|
||||
@@ -488,7 +501,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
|
||||
mapping.Data = *osdID
|
||||
|
||||
// populate the perf partition scheme entry with distributed partition details
|
||||
err := partition.PopulateDistributedPerfSchemeEntry(schemeEntry, name, perfScheme.Metadata, bluestoreConfig)
|
||||
err := PopulateDistributedPerfSchemeEntry(schemeEntry, name, perfScheme.Metadata, storeConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create distributed perf scheme entry for %s: %+v", name, err)
|
||||
}
|
||||
@@ -500,7 +513,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
|
||||
mapping.Metadata = []int{*osdID}
|
||||
|
||||
// populate the perf partition scheme entry with collocated partition details
|
||||
err := partition.PopulateCollocatedPerfSchemeEntry(schemeEntry, name, bluestoreConfig)
|
||||
err := PopulateCollocatedPerfSchemeEntry(schemeEntry, name, storeConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create collocated perf scheme entry for %s: %+v", name, err)
|
||||
}
|
||||
@@ -514,7 +527,7 @@ func getPartitionPerfScheme(context *clusterd.Context, connection client.Connect
|
||||
}
|
||||
|
||||
// determines if the given device name is already in use with existing/committed partitions
|
||||
func isDeviceInUse(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) bool {
|
||||
func isDeviceInUse(name string, nameToUUID map[string]string, scheme *PerfScheme) bool {
|
||||
parts := findPartitionsForDevice(name, nameToUUID, scheme)
|
||||
return len(parts) > 0
|
||||
}
|
||||
@@ -529,12 +542,12 @@ func isDeviceDesiredForData(mapping *DeviceOsdIDEntry) bool {
|
||||
(mapping.Data > unassignedOSDID && len(mapping.Metadata) == 1)
|
||||
}
|
||||
|
||||
func isDeviceDesiredForMetadata(mapping *DeviceOsdIDEntry, scheme *partition.PerfScheme) bool {
|
||||
func isDeviceDesiredForMetadata(mapping *DeviceOsdIDEntry, scheme *PerfScheme) bool {
|
||||
return mapping.Data == unassignedOSDID && mapping.Metadata != nil && len(mapping.Metadata) == 0
|
||||
}
|
||||
|
||||
// finds all the partition details that are on the given device name
|
||||
func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) []*partition.PerfSchemePartitionDetails {
|
||||
func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *PerfScheme) []*PerfSchemePartitionDetails {
|
||||
if scheme == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -544,7 +557,7 @@ func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *
|
||||
return nil
|
||||
}
|
||||
|
||||
parts := []*partition.PerfSchemePartitionDetails{}
|
||||
parts := []*PerfSchemePartitionDetails{}
|
||||
for _, e := range scheme.Entries {
|
||||
for _, p := range e.Partitions {
|
||||
if p.DiskUUID == diskUUID {
|
||||
@@ -558,7 +571,7 @@ func findPartitionsForDevice(name string, nameToUUID map[string]string, scheme *
|
||||
|
||||
// if a device name has changed, this function will find all partition entries with the device's static UUID and
|
||||
// then update the device name on them
|
||||
func refreshDeviceInfo(name string, nameToUUID map[string]string, scheme *partition.PerfScheme) {
|
||||
func refreshDeviceInfo(name string, nameToUUID map[string]string, scheme *PerfScheme) {
|
||||
parts := findPartitionsForDevice(name, nameToUUID, scheme)
|
||||
if len(parts) == 0 {
|
||||
return
|
||||
@@ -595,7 +608,7 @@ func (a *OsdAgent) startOSD(context *clusterd.Context, connection client.Connect
|
||||
if newOSD {
|
||||
if config.partitionScheme != nil {
|
||||
// format and partition the device if needed
|
||||
savedScheme, err := partition.LoadScheme(config.configRoot)
|
||||
savedScheme, err := LoadScheme(config.configRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", config.configRoot, err)
|
||||
}
|
||||
@@ -610,39 +623,25 @@ func (a *OsdAgent) startOSD(context *clusterd.Context, connection client.Connect
|
||||
}
|
||||
|
||||
if !skipFormat {
|
||||
err = formatDevice(context, config, a.forceFormat)
|
||||
err = formatDevice(context, config, a.forceFormat, a.storeConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed format/partition of osd %d. %+v", config.id, err)
|
||||
}
|
||||
|
||||
logger.Notice("waiting after bluestore partition/format...")
|
||||
logger.Notice("waiting after partition/format...")
|
||||
<-time.After(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// osd_data_dir/whoami does not exist yet, create/initialize the OSD
|
||||
err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, context.Executor)
|
||||
err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, a.storeConfig, context.Executor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize OSD at %s: %+v", config.rootPath, err)
|
||||
}
|
||||
|
||||
// save the osd information to applied state
|
||||
dataDiskUUID, metadataDiskUUID := "", ""
|
||||
if config.partitionScheme != nil {
|
||||
dataDiskUUID = config.partitionScheme.Partitions[partition.BlockPartitionName].DiskUUID
|
||||
metadataDiskUUID = config.partitionScheme.Partitions[partition.DatabasePartitionName].DiskUUID
|
||||
}
|
||||
|
||||
if context.EtcdClient != nil {
|
||||
settings := map[string]string{
|
||||
"path": config.configRoot,
|
||||
dataDiskUUIDKey: dataDiskUUID,
|
||||
metadataDiskUUIDKey: metadataDiskUUID,
|
||||
}
|
||||
key := path.Join(getAppliedKey(context.NodeID), fmt.Sprintf("%d", config.id))
|
||||
if err := util.StoreEtcdProperties(context.EtcdClient, key, settings); err != nil {
|
||||
return fmt.Errorf("failed to mark osd %d as applied: %+v", config.id, err)
|
||||
}
|
||||
if err := markOSDAsApplied(context, config); err != nil {
|
||||
return fmt.Errorf("failed to mark osd %d as applied: %+v", config.id, err)
|
||||
}
|
||||
} else {
|
||||
// osd_data_dir/whoami already exists, meaning the OSD is already set up.
|
||||
@@ -680,7 +679,7 @@ func (a *OsdAgent) runOSD(context *clusterd.Context, clusterName string, config
|
||||
osdUUIDArg,
|
||||
}
|
||||
|
||||
if config.dir {
|
||||
if !isBluestore(config) {
|
||||
params = append(params, fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(config.rootPath)))
|
||||
}
|
||||
|
||||
@@ -854,6 +853,40 @@ func associateOSDIDsWithMetadataDevice(etcdClient etcd.KeysAPI, nodeID, name, id
|
||||
return err
|
||||
}
|
||||
|
||||
func markOSDAsApplied(context *clusterd.Context, config *osdConfig) error {
|
||||
if context.EtcdClient == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
dataDiskUUID, metadataDiskUUID := "", ""
|
||||
if config.partitionScheme != nil {
|
||||
dataPartDetails, err := getDataPartitionDetails(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
metadataPartDetails, err := getMetadataPartitionDetails(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dataDiskUUID = dataPartDetails.DiskUUID
|
||||
metadataDiskUUID = metadataPartDetails.DiskUUID
|
||||
}
|
||||
|
||||
settings := map[string]string{
|
||||
"path": config.configRoot,
|
||||
dataDiskUUIDKey: dataDiskUUID,
|
||||
metadataDiskUUIDKey: metadataDiskUUID,
|
||||
}
|
||||
key := path.Join(getAppliedKey(context.NodeID), fmt.Sprintf("%d", config.id))
|
||||
if err := util.StoreEtcdProperties(context.EtcdClient, key, settings); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// add a device to the desired state
|
||||
func AddDesiredDevice(etcdClient etcd.KeysAPI, nodeID, uuid string, osdID int) error {
|
||||
key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), uuid)
|
||||
@@ -954,3 +987,7 @@ func loadOSDInfo(config *osdConfig) error {
|
||||
config.uuid = osdUUID
|
||||
return nil
|
||||
}
|
||||
|
||||
func isBluestore(config *osdConfig) bool {
|
||||
return !config.dir && config.partitionScheme != nil && config.partitionScheme.StoreType == Bluestore
|
||||
}
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
"github.com/google/uuid"
|
||||
testceph "github.com/rook/rook/pkg/cephmgr/client/test"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd/partition"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/clusterd/inventory"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
@@ -38,6 +37,11 @@ import (
|
||||
)
|
||||
|
||||
func TestOSDAgentWithDevices(t *testing.T) {
|
||||
testOSDAgentWithDevicesHelper(t, StoreConfig{StoreType: Bluestore})
|
||||
testOSDAgentWithDevicesHelper(t, StoreConfig{StoreType: Filestore})
|
||||
}
|
||||
|
||||
func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig StoreConfig) {
|
||||
// set up a temporary config directory that will be cleaned up after test
|
||||
configDir, err := ioutil.TempDir("", "TestOSDAgentWithDevices")
|
||||
if err != nil {
|
||||
@@ -47,7 +51,7 @@ func TestOSDAgentWithDevices(t *testing.T) {
|
||||
|
||||
clusterName := "mycluster"
|
||||
nodeID := "abc"
|
||||
etcdClient, agent, _ := createTestAgent(t, nodeID, "sdx,sdy", configDir)
|
||||
etcdClient, agent, _ := createTestAgent(t, nodeID, "sdx,sdy", configDir, &storeConfig)
|
||||
|
||||
startCount := 0
|
||||
executor := &exectest.MockExecutor{}
|
||||
@@ -74,10 +78,10 @@ func TestOSDAgentWithDevices(t *testing.T) {
|
||||
nameSuffix = parts[1]
|
||||
}
|
||||
switch {
|
||||
case execCount == 0: // first exec is the mkfs for sdy
|
||||
case execCount == 0: // first exec is the mkfs for sdx
|
||||
assert.Equal(t, "--mkfs", args[3])
|
||||
createTestKeyring(t, configDir, args)
|
||||
case execCount == 1: // all remaining execs are for partitioning sdx then mkfs sdx
|
||||
case execCount == 1: // all remaining execs are for partitioning sdy then mkfs sdy
|
||||
assert.Equal(t, "sgdisk", command)
|
||||
assert.Equal(t, "--zap-all", args[0])
|
||||
assert.Equal(t, "/dev/"+nameSuffix, args[1])
|
||||
@@ -86,10 +90,39 @@ func TestOSDAgentWithDevices(t *testing.T) {
|
||||
assert.Equal(t, "--clear", args[0])
|
||||
assert.Equal(t, "/dev/"+nameSuffix, args[2])
|
||||
case execCount == 3:
|
||||
assert.Equal(t, "/dev/"+nameSuffix, args[10])
|
||||
// the partitioning for sdy. depending on bluestore/filestore, the args will be different.
|
||||
expectedDeviceArgNum := 0
|
||||
if storeConfig.StoreType == Bluestore {
|
||||
expectedDeviceArgNum = 10
|
||||
} else {
|
||||
expectedDeviceArgNum = 4
|
||||
}
|
||||
assert.Equal(t, "sgdisk", command)
|
||||
assert.Equal(t, "/dev/"+nameSuffix, args[expectedDeviceArgNum])
|
||||
case execCount == 4:
|
||||
assert.Equal(t, "--mkfs", args[3])
|
||||
createTestKeyring(t, configDir, args)
|
||||
if storeConfig.StoreType == Bluestore {
|
||||
// the osd mkfs for sdy bluestore
|
||||
assert.Equal(t, "--mkfs", args[3])
|
||||
createTestKeyring(t, configDir, args)
|
||||
} else {
|
||||
// mkfs.ext4 for sdy filestore
|
||||
assert.Equal(t, "mkfs.ext4", command)
|
||||
}
|
||||
case execCount == 5:
|
||||
if storeConfig.StoreType == Filestore {
|
||||
// the mount for sdy filestore
|
||||
assert.Equal(t, "mount", command)
|
||||
} else {
|
||||
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
|
||||
}
|
||||
case execCount == 6:
|
||||
if storeConfig.StoreType == Filestore {
|
||||
// the osd mkfs for sdy filestore
|
||||
assert.Equal(t, "--mkfs", args[3])
|
||||
createTestKeyring(t, configDir, args)
|
||||
} else {
|
||||
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
|
||||
}
|
||||
default:
|
||||
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
|
||||
}
|
||||
@@ -136,11 +169,16 @@ func TestOSDAgentWithDevices(t *testing.T) {
|
||||
<-agent.osdsCompleted
|
||||
|
||||
assert.Equal(t, 0, agent.configCounter)
|
||||
assert.Equal(t, 5, execCount) // 1 mkfs for sdy, 3 partition steps for sdx, 1 mkfs for sdx
|
||||
assert.Equal(t, 2, outputExecCount)
|
||||
assert.Equal(t, 2, startCount) // 2 OSD procs should be started
|
||||
assert.Equal(t, 2, len(agent.osdProc), fmt.Sprintf("procs=%+v", agent.osdProc))
|
||||
|
||||
if storeConfig.StoreType == Bluestore {
|
||||
assert.Equal(t, 5, execCount) // 1 osd mkfs for sdx, 3 partition steps for sdy, 1 osd mkfs for sdy
|
||||
} else {
|
||||
assert.Equal(t, 7, execCount) // 1 osd mkfs for sdx, 3 partition steps for sdy, 1 mkfs for sdy, 1 mount for sdy, 1 osd mkfs for sdy
|
||||
}
|
||||
|
||||
err = agent.DestroyLocalService(context)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(agent.osdProc))
|
||||
@@ -159,7 +197,7 @@ func TestOSDAgentNoDevices(t *testing.T) {
|
||||
|
||||
// create a test OSD agent with no devices specified
|
||||
nodeID := "abc"
|
||||
etcdClient, agent, _ := createTestAgent(t, nodeID, "", configDir)
|
||||
etcdClient, agent, _ := createTestAgent(t, nodeID, "", configDir, nil)
|
||||
|
||||
startCount := 0
|
||||
executor := &exectest.MockExecutor{}
|
||||
@@ -251,7 +289,7 @@ func TestRemoveDevice(t *testing.T) {
|
||||
defer os.RemoveAll(configDir)
|
||||
|
||||
nodeID := "a"
|
||||
etcdClient, agent, conn := createTestAgent(t, nodeID, "", configDir)
|
||||
etcdClient, agent, conn := createTestAgent(t, nodeID, "", configDir, nil)
|
||||
executor := &exectest.MockExecutor{}
|
||||
|
||||
context := &clusterd.Context{EtcdClient: etcdClient, NodeID: nodeID, Executor: executor, ProcMan: proc.New(executor), Inventory: createInventory()}
|
||||
@@ -272,12 +310,15 @@ func TestRemoveDevice(t *testing.T) {
|
||||
assert.True(t, applied.Equals(util.CreateSet([]string{"23"})), fmt.Sprintf("applied=%+v", applied))
|
||||
}
|
||||
|
||||
func createTestAgent(t *testing.T, nodeID, devices, configDir string) (*util.MockEtcdClient, *OsdAgent, *testceph.MockConnection) {
|
||||
func createTestAgent(t *testing.T, nodeID, devices, configDir string, storeConfig *StoreConfig) (*util.MockEtcdClient, *OsdAgent, *testceph.MockConnection) {
|
||||
location := "root=here"
|
||||
forceFormat := false
|
||||
if storeConfig == nil {
|
||||
storeConfig = &StoreConfig{StoreType: Bluestore}
|
||||
}
|
||||
etcdClient := util.NewMockEtcdClient()
|
||||
factory := &testceph.MockConnectionFactory{}
|
||||
agent := NewAgent(factory, devices, "", forceFormat, location, partition.BluestoreConfig{}, nil)
|
||||
agent := NewAgent(factory, devices, false, "", "", forceFormat, location, *storeConfig, nil)
|
||||
agent.cluster = &mon.ClusterInfo{Name: "myclust"}
|
||||
agent.Initialize(&clusterd.Context{EtcdClient: etcdClient, NodeID: nodeID, ConfigDir: configDir})
|
||||
if devices == "" {
|
||||
@@ -443,7 +484,7 @@ func TestGetPartitionPerfScheme(t *testing.T) {
|
||||
return nil, "", fmt.Errorf("unexpected mon_command '%s'", string(args))
|
||||
}
|
||||
|
||||
scheme, err := getPartitionPerfScheme(context, conn, devices, partition.BluestoreConfig{})
|
||||
scheme, err := getPartitionPerfScheme(context, conn, devices, StoreConfig{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(scheme.Entries))
|
||||
|
||||
@@ -457,11 +498,11 @@ func TestGetPartitionPerfScheme(t *testing.T) {
|
||||
// and that the second is the other one.
|
||||
entry := scheme.Entries[0]
|
||||
assert.Equal(t, 11, entry.ID)
|
||||
firstBlockDevice := entry.Partitions[partition.BlockPartitionName].Device
|
||||
firstBlockDevice := entry.Partitions[BlockPartitionType].Device
|
||||
assert.True(t, firstBlockDevice == "sda" || firstBlockDevice == "sdb", firstBlockDevice)
|
||||
verifyPartitionEntry(t, entry.Partitions[partition.BlockPartitionName], firstBlockDevice, -1, 1)
|
||||
verifyPartitionEntry(t, entry.Partitions[partition.WalPartitionName], "sdc", partition.WalDefaultSizeMB, 1)
|
||||
verifyPartitionEntry(t, entry.Partitions[partition.DatabasePartitionName], "sdc", partition.DBDefaultSizeMB, 577)
|
||||
verifyPartitionEntry(t, entry.Partitions[BlockPartitionType], firstBlockDevice, -1, 1)
|
||||
verifyPartitionEntry(t, entry.Partitions[WalPartitionType], "sdc", WalDefaultSizeMB, 1)
|
||||
verifyPartitionEntry(t, entry.Partitions[DatabasePartitionType], "sdc", DBDefaultSizeMB, 577)
|
||||
|
||||
// verify the second entry in the scheme. Note the comment above about sda vs. sdb ordering.
|
||||
entry = scheme.Entries[1]
|
||||
@@ -472,9 +513,9 @@ func TestGetPartitionPerfScheme(t *testing.T) {
|
||||
} else {
|
||||
secondBlockDevice = "sda"
|
||||
}
|
||||
verifyPartitionEntry(t, entry.Partitions[partition.BlockPartitionName], secondBlockDevice, -1, 1)
|
||||
verifyPartitionEntry(t, entry.Partitions[partition.WalPartitionName], "sdc", partition.WalDefaultSizeMB, 21057)
|
||||
verifyPartitionEntry(t, entry.Partitions[partition.DatabasePartitionName], "sdc", partition.DBDefaultSizeMB, 21633)
|
||||
verifyPartitionEntry(t, entry.Partitions[BlockPartitionType], secondBlockDevice, -1, 1)
|
||||
verifyPartitionEntry(t, entry.Partitions[WalPartitionType], "sdc", WalDefaultSizeMB, 21057)
|
||||
verifyPartitionEntry(t, entry.Partitions[DatabasePartitionType], "sdc", DBDefaultSizeMB, 21633)
|
||||
}
|
||||
|
||||
func TestGetPartitionPerfSchemeDiskInUse(t *testing.T) {
|
||||
@@ -508,7 +549,7 @@ func TestGetPartitionPerfSchemeDiskInUse(t *testing.T) {
|
||||
|
||||
// get the partition scheme based on the desired devices. Since sda is already in use, the partition
|
||||
// scheme returned should reflect that.
|
||||
scheme, err := getPartitionPerfScheme(context, nil, devices, partition.BluestoreConfig{})
|
||||
scheme, err := getPartitionPerfScheme(context, nil, devices, StoreConfig{})
|
||||
assert.Nil(t, err)
|
||||
|
||||
// the partition scheme should have a single entry for osd 1 on sda and it should have collocated data and metadata
|
||||
@@ -553,20 +594,20 @@ func TestGetPartitionPerfSchemeDiskNameChanged(t *testing.T) {
|
||||
|
||||
// get the current partition scheme. This should notice that the device names changed and update the
|
||||
// partition scheme to have the latest device names
|
||||
scheme, err := getPartitionPerfScheme(context, nil, devices, partition.BluestoreConfig{})
|
||||
scheme, err := getPartitionPerfScheme(context, nil, devices, StoreConfig{})
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, scheme)
|
||||
assert.Equal(t, "nvme01-changed", scheme.Metadata.Device)
|
||||
assert.Equal(t, "sda-changed", scheme.Entries[0].Partitions[partition.BlockPartitionName].Device)
|
||||
assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[partition.WalPartitionName].Device)
|
||||
assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[partition.DatabasePartitionName].Device)
|
||||
assert.Equal(t, "sda-changed", scheme.Entries[0].Partitions[BlockPartitionType].Device)
|
||||
assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[WalPartitionType].Device)
|
||||
assert.Equal(t, "nvme01-changed", scheme.Entries[0].Partitions[DatabasePartitionType].Device)
|
||||
}
|
||||
|
||||
func createInventory() *inventory.Config {
|
||||
return &inventory.Config{Local: &inventory.Hardware{Disks: []*inventory.LocalDisk{}}}
|
||||
}
|
||||
|
||||
func verifyPartitionEntry(t *testing.T, actual *partition.PerfSchemePartitionDetails, expectedDevice string,
|
||||
func verifyPartitionEntry(t *testing.T, actual *PerfSchemePartitionDetails, expectedDevice string,
|
||||
expectedSize int64, expectedOffset int64) {
|
||||
|
||||
assert.Equal(t, expectedDevice, actual.Device)
|
||||
@@ -574,12 +615,12 @@ func verifyPartitionEntry(t *testing.T, actual *partition.PerfSchemePartitionDet
|
||||
assert.Equal(t, expectedOffset, actual.OffsetMB)
|
||||
}
|
||||
|
||||
func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string) (entry *partition.PerfSchemeEntry, diskUUID string) {
|
||||
entry = partition.NewPerfSchemeEntry()
|
||||
func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string) (entry *PerfSchemeEntry, diskUUID string) {
|
||||
entry = NewPerfSchemeEntry(Bluestore)
|
||||
entry.ID = osdID
|
||||
entry.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
partition.PopulateCollocatedPerfSchemeEntry(entry, device, partition.BluestoreConfig{})
|
||||
scheme := partition.NewPerfScheme()
|
||||
PopulateCollocatedPerfSchemeEntry(entry, device, StoreConfig{})
|
||||
scheme := NewPerfScheme()
|
||||
scheme.Entries = append(scheme.Entries, entry)
|
||||
err := scheme.Save(configDir)
|
||||
assert.Nil(t, err)
|
||||
@@ -594,19 +635,19 @@ func mockPartitionSchemeEntry(t *testing.T, osdID int, device, configDir string)
|
||||
return entry, diskUUID
|
||||
}
|
||||
|
||||
func mockDistributedPartitionScheme(t *testing.T, osdID int, metadataDevice, device, configDir string) (*partition.PerfScheme, string, string) {
|
||||
scheme := partition.NewPerfScheme()
|
||||
scheme.Metadata = partition.NewMetadataDeviceInfo(metadataDevice)
|
||||
func mockDistributedPartitionScheme(t *testing.T, osdID int, metadataDevice, device, configDir string) (*PerfScheme, string, string) {
|
||||
scheme := NewPerfScheme()
|
||||
scheme.Metadata = NewMetadataDeviceInfo(metadataDevice)
|
||||
|
||||
entry := partition.NewPerfSchemeEntry()
|
||||
entry := NewPerfSchemeEntry(Bluestore)
|
||||
entry.ID = osdID
|
||||
entry.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
|
||||
partition.PopulateDistributedPerfSchemeEntry(entry, device, scheme.Metadata, partition.BluestoreConfig{})
|
||||
PopulateDistributedPerfSchemeEntry(entry, device, scheme.Metadata, StoreConfig{})
|
||||
scheme.Entries = append(scheme.Entries, entry)
|
||||
err := scheme.Save(configDir)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// return the full partition scheme, the metadata device UUID and the data device UUID
|
||||
return scheme, scheme.Metadata.DiskUUID, entry.Partitions[partition.BlockPartitionName].DiskUUID
|
||||
return scheme, scheme.Metadata.DiskUUID, entry.Partitions[BlockPartitionType].DiskUUID
|
||||
}
|
||||
@@ -49,14 +49,14 @@ func formatLocation(location string) ([]string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if !isCrushFieldSet("hostName", pairs) {
|
||||
if !isCrushFieldSet("host", pairs) {
|
||||
// host name isn't set yet, attempt to set a default
|
||||
hostName, err := os.Hostname()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get hostname, %+v", err)
|
||||
}
|
||||
|
||||
pairs = append(pairs, formatProperty("hostName", hostName))
|
||||
pairs = append(pairs, formatProperty("host", hostName))
|
||||
}
|
||||
|
||||
// set a default root if it's not already set
|
||||
|
||||
@@ -49,12 +49,12 @@ func TestCrushMap(t *testing.T) {
|
||||
argsSet := util.CreateSet(request.Args)
|
||||
assert.True(t, argsSet.Contains("root=default"))
|
||||
assert.True(t, argsSet.Contains("dc=datacenter1"))
|
||||
assert.True(t, argsSet.Contains("hostName=node1"))
|
||||
assert.True(t, argsSet.Contains("host=node1"))
|
||||
|
||||
return []byte{}, "", nil
|
||||
}
|
||||
|
||||
location := "root=default,dc=datacenter1,hostName=node1"
|
||||
location := "root=default,dc=datacenter1,host=node1"
|
||||
|
||||
err := addOSDToCrushMap(conn, context, 23, "/", location)
|
||||
assert.Nil(t, err)
|
||||
@@ -84,17 +84,17 @@ func TestCrushLocation(t *testing.T) {
|
||||
locSet := util.CreateSet(res)
|
||||
assert.True(t, locSet.Contains("root=default"))
|
||||
assert.True(t, locSet.Contains("dc=datacenter1"))
|
||||
assert.True(t, locSet.Contains(fmt.Sprintf("hostName=%s", hostName)))
|
||||
assert.True(t, locSet.Contains(fmt.Sprintf("host=%s", hostName)))
|
||||
|
||||
// test that if host name and root are already set they will be honored
|
||||
loc = "root=otherRoot,dc=datacenter2,hostName=node123"
|
||||
loc = "root=otherRoot,dc=datacenter2,host=node123"
|
||||
res, err = formatLocation(loc)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, len(res))
|
||||
locSet = util.CreateSet(res)
|
||||
assert.True(t, locSet.Contains("root=otherRoot"))
|
||||
assert.True(t, locSet.Contains("dc=datacenter2"))
|
||||
assert.True(t, locSet.Contains("hostName=node123"))
|
||||
assert.True(t, locSet.Contains("host=node123"))
|
||||
|
||||
// test an invalid CRUSH location format
|
||||
loc = "root=default,prop:value"
|
||||
|
||||
+68
-19
@@ -25,6 +25,8 @@ import (
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"strings"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/clusterd/inventory"
|
||||
@@ -62,7 +64,7 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
|
||||
}
|
||||
|
||||
// initialize the desired osds
|
||||
devices, err := getAvailableDevices(dcontext, hardware.Disks, agent.devices)
|
||||
devices, err := getAvailableDevices(dcontext, hardware.Disks, agent.devices, agent.usingDeviceFilter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get available devices. %+v", err)
|
||||
}
|
||||
@@ -74,8 +76,8 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
|
||||
}
|
||||
|
||||
// initialize the data directories, with the default dir if no devices were specified
|
||||
useDataDir := len(devices.Entries) == 0
|
||||
dirs, err := getDataDirs(dcontext, useDataDir)
|
||||
devicesSpecified := len(devices.Entries) > 0
|
||||
dirs, err := getDataDirs(dcontext, agent.directories, devicesSpecified)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get data dirs. %+v", err)
|
||||
}
|
||||
@@ -95,7 +97,14 @@ func Run(dcontext *clusterd.DaemonContext, agent *OsdAgent) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk, desiredDevices string) (*DeviceOsdMapping, error) {
|
||||
func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.LocalDisk, desiredDevices string,
|
||||
usingDeviceFilter bool) (*DeviceOsdMapping, error) {
|
||||
|
||||
var deviceList []string
|
||||
if !usingDeviceFilter {
|
||||
deviceList = strings.Split(desiredDevices, ",")
|
||||
}
|
||||
|
||||
available := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}}
|
||||
for _, device := range devices {
|
||||
if device.Type == sys.PartType {
|
||||
@@ -109,12 +118,24 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L
|
||||
if desiredDevices == "all" {
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
|
||||
} else if desiredDevices != "" {
|
||||
// the desired devices is a regular expression
|
||||
matched, err := regexp.Match(desiredDevices, []byte(device.Name))
|
||||
var matched bool
|
||||
var err error
|
||||
if usingDeviceFilter {
|
||||
// the desired devices is a regular expression
|
||||
matched, err = regexp.Match(desiredDevices, []byte(device.Name))
|
||||
} else {
|
||||
for i := range deviceList {
|
||||
if device.Name == deviceList[i] {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil && matched {
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID}
|
||||
} else {
|
||||
logger.Infof("skipping device %s that does not match the regular expression %s. %+v", device.Name, desiredDevices, err)
|
||||
logger.Infof("skipping device %s that does not match the device filter/list `%s`. %+v", device.Name, desiredDevices, err)
|
||||
}
|
||||
|
||||
} else {
|
||||
@@ -126,28 +147,56 @@ func getAvailableDevices(context *clusterd.DaemonContext, devices []*inventory.L
|
||||
return available, nil
|
||||
}
|
||||
|
||||
func getDataDirs(context *clusterd.DaemonContext, useDataDir bool) (map[string]int, error) {
|
||||
filePath := path.Join(context.ConfigDir, dirOSDConfigFilename)
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
// the config file doesn't exist yet, just return the empty map unless the data dir is desired
|
||||
if useDataDir {
|
||||
return map[string]int{context.ConfigDir: unassignedOSDID}, nil
|
||||
}
|
||||
|
||||
return map[string]int{}, nil
|
||||
func getDataDirs(context *clusterd.DaemonContext, desiredDirs string, devicesSpecified bool) (map[string]int, error) {
|
||||
var dirList []string
|
||||
if desiredDirs != "" {
|
||||
dirList = strings.Split(desiredDirs, ",")
|
||||
}
|
||||
|
||||
filePath := path.Join(context.ConfigDir, dirOSDConfigFilename)
|
||||
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
// the config file doesn't exist yet
|
||||
if len(dirList) == 0 {
|
||||
if devicesSpecified {
|
||||
// no dirs desired, user is using devices instead
|
||||
return map[string]int{}, nil
|
||||
} else {
|
||||
// no devices or dirs specified, return the default data dir
|
||||
return map[string]int{context.ConfigDir: unassignedOSDID}, nil
|
||||
}
|
||||
}
|
||||
|
||||
dirMap := make(map[string]int, len(dirList))
|
||||
addDirsToDirMap(dirList, &dirMap)
|
||||
return dirMap, nil
|
||||
}
|
||||
|
||||
// read the saved dir map from disk
|
||||
var dirMap map[string]int
|
||||
b, err := ioutil.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var dirs map[string]int
|
||||
err = json.Unmarshal(b, &dirs)
|
||||
err = json.Unmarshal(b, &dirMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dirs, nil
|
||||
|
||||
// if the user has specified any directories to use, merge them into the saved config
|
||||
addDirsToDirMap(dirList, &dirMap)
|
||||
|
||||
return dirMap, nil
|
||||
}
|
||||
|
||||
func addDirsToDirMap(dirList []string, dirMap *map[string]int) {
|
||||
for _, d := range dirList {
|
||||
if _, ok := (*dirMap)[d]; !ok {
|
||||
// the users dir isn't already in the map, add it with an unassigned ID
|
||||
(*dirMap)[d] = unassignedOSDID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func saveDirConfig(context *clusterd.DaemonContext, config map[string]int) error {
|
||||
|
||||
@@ -35,33 +35,51 @@ func TestStoreOSDDirMap(t *testing.T) {
|
||||
defer os.RemoveAll(context.ConfigDir)
|
||||
os.MkdirAll(context.ConfigDir, 0755)
|
||||
|
||||
dirMap, err := getDataDirs(context, false)
|
||||
// user has specified devices to use, no dirs should be returned
|
||||
dirMap, err := getDataDirs(context, "", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(dirMap))
|
||||
|
||||
dirMap, err = getDataDirs(context, true)
|
||||
// user has no devices specified, should return default dir
|
||||
dirMap, err = getDataDirs(context, "", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(dirMap))
|
||||
assert.Equal(t, unassignedOSDID, dirMap[context.ConfigDir])
|
||||
dirMap[context.ConfigDir] = 0
|
||||
|
||||
// user has no devices specified but does specify dirs, those should be returned
|
||||
dirMap, err = getDataDirs(context, "/rook/dir1", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(dirMap))
|
||||
assert.Equal(t, unassignedOSDID, dirMap["/rook/dir1"])
|
||||
dirMap["/rook/dir1"] = 0 // simulate an OSD ID being assigned to the dir
|
||||
|
||||
// save the directory config
|
||||
err = saveDirConfig(context, dirMap)
|
||||
assert.Nil(t, err)
|
||||
|
||||
dirMap, err = getDataDirs(context, false)
|
||||
// user has specified devices to use, we should still return the saved dir
|
||||
dirMap, err = getDataDirs(context, "", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(dirMap))
|
||||
assert.Equal(t, 0, dirMap[context.ConfigDir])
|
||||
assert.Equal(t, 0, dirMap["/rook/dir1"])
|
||||
|
||||
// add another directory to the map
|
||||
// user has specified devices and also a directory to use. it should be added to the dir map
|
||||
dirMap, err = getDataDirs(context, "/tmp/mydir", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(dirMap))
|
||||
assert.Equal(t, 0, dirMap["/rook/dir1"])
|
||||
assert.Equal(t, unassignedOSDID, dirMap["/tmp/mydir"])
|
||||
|
||||
// simulate that the user's dir got an OSD by assigning it an ID
|
||||
dirMap["/tmp/mydir"] = 23
|
||||
err = saveDirConfig(context, dirMap)
|
||||
assert.Nil(t, err)
|
||||
|
||||
dirMap, err = getDataDirs(context, false)
|
||||
// user is still specifying the directory, we should get back it's ID now
|
||||
dirMap, err = getDataDirs(context, "/tmp/mydir", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(dirMap))
|
||||
assert.Equal(t, 0, dirMap[context.ConfigDir])
|
||||
assert.Equal(t, 0, dirMap["/rook/dir1"])
|
||||
assert.Equal(t, 23, dirMap["/tmp/mydir"])
|
||||
}
|
||||
|
||||
@@ -99,7 +117,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="MY-PART"`, nil
|
||||
&inventory.LocalDisk{Name: "rdb"},
|
||||
}
|
||||
// select all devices
|
||||
mapping, err := getAvailableDevices(context, devices, "all")
|
||||
mapping, err := getAvailableDevices(context, devices, "all", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 4, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sda"].Data)
|
||||
@@ -107,26 +125,30 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb" PARTLABEL="MY-PART"`, nil
|
||||
assert.Equal(t, -1, mapping.Entries["rda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["rdb"].Data)
|
||||
|
||||
// select no devices
|
||||
mapping, err = getAvailableDevices(context, devices, "")
|
||||
// select no devices both using and not using a filter
|
||||
mapping, err = getAvailableDevices(context, devices, "", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(mapping.Entries))
|
||||
|
||||
mapping, err = getAvailableDevices(context, devices, "", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(mapping.Entries))
|
||||
|
||||
// select the sda devices
|
||||
mapping, err = getAvailableDevices(context, devices, "^sd.$")
|
||||
mapping, err = getAvailableDevices(context, devices, "^sd.$", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sda"].Data)
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
|
||||
// select an exact device
|
||||
mapping, err = getAvailableDevices(context, devices, "sdd")
|
||||
mapping, err = getAvailableDevices(context, devices, "sdd", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["sdd"].Data)
|
||||
|
||||
// select all devices except those that have a prefix of "s"
|
||||
mapping, err = getAvailableDevices(context, devices, "^[^s]")
|
||||
mapping, err = getAvailableDevices(context, devices, "^[^s]", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(mapping.Entries))
|
||||
assert.Equal(t, -1, mapping.Entries["rda"].Data)
|
||||
|
||||
+125
-50
@@ -25,12 +25,12 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/client"
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd/partition"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/clusterd/inventory"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
@@ -41,6 +41,7 @@ import (
|
||||
const (
|
||||
DevicesValue = "devices"
|
||||
ForceFormatValue = "forceFormat"
|
||||
diskByPartUUID = "/dev/disk/by-partuuid"
|
||||
cephOsdKey = mon.CephKey + "/osd"
|
||||
desiredOsdRootKey = cephOsdKey + "/" + clusterd.DesiredKey + "/%s"
|
||||
deviceDesiredKey = desiredOsdRootKey + "/device"
|
||||
@@ -58,7 +59,7 @@ type osdConfig struct {
|
||||
id int
|
||||
uuid uuid.UUID
|
||||
dir bool
|
||||
partitionScheme *partition.PerfSchemeEntry
|
||||
partitionScheme *PerfSchemeEntry
|
||||
}
|
||||
|
||||
type Device struct {
|
||||
@@ -83,42 +84,42 @@ type DeviceOsdIDEntry struct {
|
||||
}
|
||||
|
||||
// format the given device for usage by an OSD
|
||||
func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool) error {
|
||||
blockDetails, err := getBlockPartitionDetails(config)
|
||||
func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool, storeConfig StoreConfig) error {
|
||||
dataDetails, err := getDataPartitionDetails(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check if partitions belong to rook
|
||||
ownPartitions, devFS, err := checkIfDeviceAvailable(context.Executor, blockDetails.Device)
|
||||
ownPartitions, devFS, err := checkIfDeviceAvailable(context.Executor, dataDetails.Device)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to format device. %+v", err)
|
||||
}
|
||||
|
||||
if !ownPartitions {
|
||||
if forceFormat {
|
||||
logger.Warningf("device %s is being formatted, but has partitions!!", blockDetails.Device)
|
||||
logger.Warningf("device %s is being formatted, but has partitions!!", dataDetails.Device)
|
||||
} else {
|
||||
logger.Warningf("device %s has partitions that will not be formatted. Skipping device.", blockDetails.Device)
|
||||
logger.Warningf("device %s has partitions that will not be formatted. Skipping device.", dataDetails.Device)
|
||||
}
|
||||
}
|
||||
|
||||
if devFS != "" {
|
||||
if forceFormat {
|
||||
// there's a filesystem on the device, but the user has specified to force a format. give a warning about that.
|
||||
logger.Warningf("device %s already formatted with %s, but forcing a format!!!", blockDetails.Device, devFS)
|
||||
logger.Warningf("device %s already formatted with %s, but forcing a format!!!", dataDetails.Device, devFS)
|
||||
} else {
|
||||
// disk is already formatted and the user doesn't want to force it, but we require partitioning
|
||||
return fmt.Errorf("device %s already formatted with %s", blockDetails.Device, devFS)
|
||||
return fmt.Errorf("device %s already formatted with %s", dataDetails.Device, devFS)
|
||||
}
|
||||
}
|
||||
|
||||
// format the device
|
||||
dangerousToFormat := !ownPartitions || devFS != ""
|
||||
if !dangerousToFormat || forceFormat {
|
||||
err := partitionBluestoreOSD(context, config)
|
||||
err := partitionOSD(context, config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to partion device %s. %v", blockDetails.Device, err)
|
||||
return fmt.Errorf("failed to partion device %s. %v", dataDetails.Device, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,14 +158,14 @@ func rookOwnsPartitions(partitions []*sys.Partition) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// partitions a given device exclusively for bluestore metadata usage
|
||||
func partitionBluestoreMetadata(context *clusterd.Context, info *partition.MetadataDeviceInfo, configRoot string) error {
|
||||
// partitions a given device exclusively for metadata usage
|
||||
func partitionMetadata(context *clusterd.Context, info *MetadataDeviceInfo, configRoot string) error {
|
||||
if len(info.Partitions) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// check to see if the metadata partition scheme has already been applied
|
||||
savedScheme, err := partition.LoadScheme(configRoot)
|
||||
savedScheme, err := LoadScheme(configRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", configRoot, err)
|
||||
}
|
||||
@@ -212,28 +213,53 @@ func partitionBluestoreMetadata(context *clusterd.Context, info *partition.Metad
|
||||
return nil
|
||||
}
|
||||
|
||||
// Partitions a device for use by a bluestore osd.
|
||||
// Partitions a device for use by a osd.
|
||||
// If there are any partitions or formatting already on the device, it will be wiped.
|
||||
func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error {
|
||||
blockDetails, err := getBlockPartitionDetails(config)
|
||||
func partitionOSD(context *clusterd.Context, config *osdConfig) error {
|
||||
dataDetails, err := getDataPartitionDetails(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// zap/clear all existing partitions on the device
|
||||
err = sys.RemovePartitions(blockDetails.Device, context.Executor)
|
||||
err = sys.RemovePartitions(dataDetails.Device, context.Executor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to zap partitions on metadata device /dev/%s: %+v", blockDetails.Device, err)
|
||||
return fmt.Errorf("failed to zap partitions on metadata device /dev/%s: %+v", dataDetails.Device, err)
|
||||
}
|
||||
|
||||
// create the partitions on the device
|
||||
err = sys.CreatePartitions(blockDetails.Device, config.partitionScheme.GetPartitionArgs(), context.Executor)
|
||||
err = sys.CreatePartitions(dataDetails.Device, config.partitionScheme.GetPartitionArgs(), context.Executor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to partition /dev/%s. %+v", blockDetails.Device, err)
|
||||
return fmt.Errorf("failed to partition /dev/%s. %+v", dataDetails.Device, err)
|
||||
}
|
||||
|
||||
if config.partitionScheme.StoreType == Filestore {
|
||||
// the OSD is using filestore, create a filesystem for the device and mount it under config root
|
||||
dataPartDetails := config.partitionScheme.Partitions[FilestoreDataPartitionType]
|
||||
dataPartPath := filepath.Join(diskByPartUUID, dataPartDetails.PartitionUUID)
|
||||
logger.Infof("waiting for partition path %s", dataPartPath)
|
||||
err = waitForPath(dataPartPath, context.Executor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed waiting for %s: %+v", dataPartPath, err)
|
||||
}
|
||||
err = sys.FormatDevice(dataPartPath, context.Executor)
|
||||
if err != nil {
|
||||
logger.Warningf("first attempt to format partition %s on device %s failed. Waiting 2 seconds then retrying: %+v",
|
||||
dataPartDetails.PartitionUUID, dataPartDetails.Device, err)
|
||||
<-time.After(2 * time.Second)
|
||||
err = sys.FormatDevice(dataPartPath, context.Executor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to format partition %s on device %s. %+v", dataPartDetails.PartitionUUID, dataPartDetails.Device, err)
|
||||
}
|
||||
}
|
||||
err = sys.MountDevice(dataPartPath, config.rootPath, context.Executor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to mount %s at %s: %+v", dataPartPath, config.configRoot, context.Executor)
|
||||
}
|
||||
}
|
||||
|
||||
// save the partition scheme entry to disk now that it has been committed
|
||||
savedScheme, err := partition.LoadScheme(config.configRoot)
|
||||
savedScheme, err := LoadScheme(config.configRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load the saved partition scheme from %s: %+v", config.configRoot, err)
|
||||
}
|
||||
@@ -243,27 +269,27 @@ func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error {
|
||||
}
|
||||
|
||||
// update the uuid of the disk in the inventory in memory
|
||||
logger.Debugf("Updating disk uuid %s on device %s", blockDetails.DiskUUID, blockDetails.Device)
|
||||
logger.Debugf("Updating disk uuid %s on device %s", dataDetails.DiskUUID, dataDetails.Device)
|
||||
for _, disk := range context.Inventory.Local.Disks {
|
||||
if disk.Name == blockDetails.Device {
|
||||
logger.Debugf("Updated uuid on device %s", blockDetails.Device)
|
||||
disk.UUID = blockDetails.DiskUUID
|
||||
if disk.Name == dataDetails.Device {
|
||||
logger.Debugf("Updated uuid on device %s", dataDetails.Device)
|
||||
disk.UUID = dataDetails.DiskUUID
|
||||
}
|
||||
}
|
||||
|
||||
// save the desired state of the osd for this device
|
||||
if context.EtcdClient != nil {
|
||||
err = associateOsdIDWithDevice(context.EtcdClient, context.NodeID, blockDetails.DiskUUID, config.id, false)
|
||||
err = associateOsdIDWithDevice(context.EtcdClient, context.NodeID, dataDetails.DiskUUID, config.id, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to associate osd id %d with device %s (%s)", config.id, blockDetails.Device, blockDetails.DiskUUID)
|
||||
return fmt.Errorf("failed to associate osd id %d with device %s (%s)", config.id, dataDetails.Device, dataDetails.DiskUUID)
|
||||
}
|
||||
if config.partitionScheme.IsCollocated() {
|
||||
// the metadata is on the same disk as the data, associate the osd ID with the device for metadata too
|
||||
err = associateOSDIDsWithMetadataDevice(
|
||||
context.EtcdClient, context.NodeID, blockDetails.DiskUUID, fmt.Sprintf("%d", config.id))
|
||||
context.EtcdClient, context.NodeID, dataDetails.DiskUUID, fmt.Sprintf("%d", config.id))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to associate osd id %d with device %s (%s) for metadata",
|
||||
config.id, blockDetails.Device, blockDetails.DiskUUID)
|
||||
config.id, dataDetails.Device, dataDetails.DiskUUID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -271,17 +297,39 @@ func partitionBluestoreOSD(context *clusterd.Context, config *osdConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func getBlockPartitionDetails(config *osdConfig) (*partition.PerfSchemePartitionDetails, error) {
|
||||
func getDataPartitionDetails(config *osdConfig) (*PerfSchemePartitionDetails, error) {
|
||||
if config.partitionScheme == nil {
|
||||
return nil, fmt.Errorf("partition scheme missing from %+v", config)
|
||||
}
|
||||
|
||||
blockDetails, ok := config.partitionScheme.Partitions[partition.BlockPartitionName]
|
||||
if !ok || blockDetails == nil {
|
||||
return nil, fmt.Errorf("block partition missing from %+v", config.partitionScheme)
|
||||
dataPartitionType := config.partitionScheme.getDataPartitionType()
|
||||
|
||||
dataDetails, ok := config.partitionScheme.Partitions[dataPartitionType]
|
||||
if !ok || dataDetails == nil {
|
||||
return nil, fmt.Errorf("data partition missing from %+v", config.partitionScheme)
|
||||
}
|
||||
|
||||
return blockDetails, nil
|
||||
return dataDetails, nil
|
||||
}
|
||||
|
||||
func getMetadataPartitionDetails(config *osdConfig) (*PerfSchemePartitionDetails, error) {
|
||||
if config.partitionScheme == nil {
|
||||
return nil, fmt.Errorf("partition scheme missing from %+v", config)
|
||||
}
|
||||
|
||||
metadataPartitionType := config.partitionScheme.getMetadataPartitionType()
|
||||
|
||||
if config.partitionScheme.StoreType == Filestore {
|
||||
// TODO: support separate metadata device for filestore (just use the data partition details for now)
|
||||
return getDataPartitionDetails(config)
|
||||
}
|
||||
|
||||
metadataDetails, ok := config.partitionScheme.Partitions[metadataPartitionType]
|
||||
if !ok || metadataDetails == nil {
|
||||
return nil, fmt.Errorf("metadata partition missing from %+v", config.partitionScheme)
|
||||
}
|
||||
|
||||
return metadataDetails, nil
|
||||
}
|
||||
|
||||
func getDiskSize(context *clusterd.Context, name string) (uint64, error) {
|
||||
@@ -294,6 +342,24 @@ func getDiskSize(context *clusterd.Context, name string) (uint64, error) {
|
||||
return 0, fmt.Errorf("device %s not found", name)
|
||||
}
|
||||
|
||||
func waitForPath(path string, executor exec.Executor) error {
|
||||
retryCount := 0
|
||||
retryMax := 25
|
||||
sleepTime := 250
|
||||
for {
|
||||
_, err := executor.ExecuteStat(path)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
retryCount++
|
||||
if retryCount > retryMax {
|
||||
return err
|
||||
}
|
||||
<-time.After(time.Duration(sleepTime) * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func registerOSD(bootstrapConn client.Connection) (*int, *uuid.UUID, error) {
|
||||
var err error
|
||||
osdUUID, err := uuid.NewRandom()
|
||||
@@ -311,9 +377,15 @@ func registerOSD(bootstrapConn client.Connection) (*int, *uuid.UUID, error) {
|
||||
return &osdID, &osdUUID, nil
|
||||
}
|
||||
|
||||
func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]string, error) {
|
||||
func getStoreSettings(context *clusterd.Context, config *osdConfig, storeConfig StoreConfig) (map[string]string, error) {
|
||||
settings := map[string]string{}
|
||||
if config.dir {
|
||||
if config.dir || (config.partitionScheme != nil && config.partitionScheme.StoreType == Filestore) {
|
||||
// add additional filestore settings for dirs and filestore devices
|
||||
journalSize := JournalDefaultSizeMB
|
||||
if storeConfig.JournalSizeMB > 0 {
|
||||
journalSize = storeConfig.JournalSizeMB
|
||||
}
|
||||
settings["osd journal size"] = strconv.Itoa(journalSize)
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
@@ -321,21 +393,22 @@ func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]
|
||||
return nil, fmt.Errorf("failed to find partitions from config for osd %d", config.id)
|
||||
}
|
||||
|
||||
// add additional bluestore settings
|
||||
parts := config.partitionScheme.Partitions
|
||||
walPartition, ok := parts[partition.WalPartitionName]
|
||||
walPartition, ok := parts[WalPartitionType]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("failed to find wal partition for osd %d", config.id)
|
||||
}
|
||||
dbPartition, ok := parts[partition.DatabasePartitionName]
|
||||
dbPartition, ok := parts[DatabasePartitionType]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("failed to find db partition for osd %d", config.id)
|
||||
}
|
||||
blockPartition, ok := parts[partition.BlockPartitionName]
|
||||
blockPartition, ok := parts[BlockPartitionType]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("failed to find block partition for osd %d", config.id)
|
||||
}
|
||||
|
||||
prefix := "/dev/disk/by-partuuid"
|
||||
prefix := diskByPartUUID
|
||||
settings["bluestore block wal path"] = path.Join(prefix, walPartition.PartitionUUID)
|
||||
settings["bluestore block db path"] = path.Join(prefix, dbPartition.PartitionUUID)
|
||||
settings["bluestore block path"] = path.Join(prefix, blockPartition.PartitionUUID)
|
||||
@@ -343,12 +416,12 @@ func getStoreSettings(context *clusterd.Context, config *osdConfig) (map[string]
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context,
|
||||
bootstrapConn client.Connection, cluster *mon.ClusterInfo, location string, executor exec.Executor) error {
|
||||
func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context, bootstrapConn client.Connection,
|
||||
cluster *mon.ClusterInfo, location string, storeConfig StoreConfig, executor exec.Executor) error {
|
||||
|
||||
cephConfig := mon.CreateDefaultCephConfig(context, cluster, config.rootPath, !config.dir)
|
||||
cephConfig := mon.CreateDefaultCephConfig(context, cluster, config.rootPath, isBluestore(config))
|
||||
|
||||
if config.dir {
|
||||
if !isBluestore(config) {
|
||||
// using the local file system requires some config overrides
|
||||
// http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/#not-recommended
|
||||
cephConfig.GlobalConfig.OsdMaxObjectNameLen = 256
|
||||
@@ -356,7 +429,7 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context
|
||||
}
|
||||
|
||||
// bluestore has some extra settings
|
||||
settings, err := getStoreSettings(context, config)
|
||||
settings, err := getStoreSettings(context, config, storeConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read store settings. %+v", err)
|
||||
}
|
||||
@@ -364,7 +437,7 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context
|
||||
// write the OSD config file to disk
|
||||
keyringPath := getOSDKeyringPath(config.rootPath)
|
||||
_, err = mon.GenerateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id),
|
||||
keyringPath, !config.dir, cephConfig, settings)
|
||||
keyringPath, isBluestore(config), cephConfig, settings)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write OSD %d config file: %+v", config.id, err)
|
||||
}
|
||||
@@ -461,7 +534,7 @@ func createOSDFileSystem(context *clusterd.Context, clusterName string, config *
|
||||
fmt.Sprintf("--monmap=%s", monMapTmpPath),
|
||||
}
|
||||
|
||||
if config.dir {
|
||||
if !isBluestore(config) {
|
||||
options = append(options, fmt.Sprintf("--osd-journal=%s", getOSDJournalPath(config.rootPath)))
|
||||
options = append(options, fmt.Sprintf("--keyring=%s", getOSDKeyringPath(config.rootPath)))
|
||||
}
|
||||
@@ -523,8 +596,6 @@ func addOSDToCrushMap(osdConn client.Connection, context *clusterd.Context, osdI
|
||||
weight, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", weight), 64)
|
||||
|
||||
osdEntity := fmt.Sprintf("osd.%d", osdID)
|
||||
logger.Infof("OSD %s at %s, bytes: %d, weight: %.4f", osdEntity, osdDataPath, all, weight)
|
||||
|
||||
locArgs, err := formatLocation(location)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -536,6 +607,10 @@ func addOSDToCrushMap(osdConn client.Connection, context *clusterd.Context, osdI
|
||||
"weight": weight,
|
||||
"args": locArgs,
|
||||
}
|
||||
|
||||
logger.Infof("adding %s (%s), bytes: %d, weight: %.4f, to crush map at '%s'",
|
||||
osdEntity, osdDataPath, all, weight, strings.Join(locArgs, " "))
|
||||
|
||||
_, err = client.ExecuteMonCommand(osdConn, cmd, fmt.Sprintf("adding %s to crush map", osdEntity))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed adding %s to crush map: %+v", osdEntity, err)
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
testceph "github.com/rook/rook/pkg/cephmgr/client/test"
|
||||
"github.com/rook/rook/pkg/cephmgr/osd/partition"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/clusterd/inventory"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
@@ -133,11 +132,13 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda" PARTLABEL="ROOK-OSD0-BLOCK"`
|
||||
return output, nil
|
||||
}
|
||||
|
||||
storeConfig := StoreConfig{StoreType: Bluestore}
|
||||
|
||||
// set up a partition scheme entry for sda (collocated metadata and data)
|
||||
entry := partition.NewPerfSchemeEntry()
|
||||
entry := NewPerfSchemeEntry(storeConfig.StoreType)
|
||||
entry.ID = 1
|
||||
entry.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
partition.PopulateCollocatedPerfSchemeEntry(entry, "sda", partition.BluestoreConfig{})
|
||||
PopulateCollocatedPerfSchemeEntry(entry, "sda", storeConfig)
|
||||
|
||||
context := &clusterd.Context{EtcdClient: etcdClient, Executor: executor, NodeID: nodeID,
|
||||
ConfigDir: configDir, Inventory: createInventory()}
|
||||
@@ -154,7 +155,7 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda" PARTLABEL="ROOK-OSD0-BLOCK"`
|
||||
|
||||
// try to format the device. even though the device has existing partitions, they are owned by rook, so it is safe
|
||||
// to format and the format/partitioning will happen.
|
||||
err = formatDevice(context, config, false)
|
||||
err = formatDevice(context, config, false, storeConfig)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, execCount)
|
||||
assert.Equal(t, 3, outputExecCount)
|
||||
@@ -195,21 +196,21 @@ func TestPartitionBluestoreMetadata(t *testing.T) {
|
||||
context := &clusterd.Context{EtcdClient: etcdClient, Executor: executor, NodeID: nodeID, ConfigDir: configDir}
|
||||
|
||||
// create metadata partition information for 2 OSDs (sdb, sdc) storing their metadata on device sda
|
||||
bluestoreConfig := partition.BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
|
||||
metadata := partition.NewMetadataDeviceInfo("sda")
|
||||
storeConfig := StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2}
|
||||
metadata := NewMetadataDeviceInfo("sda")
|
||||
|
||||
e1 := partition.NewPerfSchemeEntry()
|
||||
e1 := NewPerfSchemeEntry(Bluestore)
|
||||
e1.ID = 1
|
||||
e1.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
partition.PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, bluestoreConfig)
|
||||
PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, storeConfig)
|
||||
|
||||
e2 := partition.NewPerfSchemeEntry()
|
||||
e2 := NewPerfSchemeEntry(Bluestore)
|
||||
e2.ID = 2
|
||||
e2.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
partition.PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, bluestoreConfig)
|
||||
PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, storeConfig)
|
||||
|
||||
// perform the metadata device partition
|
||||
err = partitionBluestoreMetadata(context, metadata, configDir)
|
||||
err = partitionMetadata(context, metadata, configDir)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, execCount)
|
||||
|
||||
@@ -221,7 +222,12 @@ func TestPartitionBluestoreMetadata(t *testing.T) {
|
||||
assert.True(t, util.CreateSet(desiredIds).Equals(util.CreateSet([]string{"1", "2"})))
|
||||
}
|
||||
|
||||
func TestPartitionBluestoreOSD(t *testing.T) {
|
||||
func TestPartitionOSD(t *testing.T) {
|
||||
testPartitionOSDHelper(t, StoreConfig{StoreType: Bluestore, WalSizeMB: 1, DatabaseSizeMB: 2})
|
||||
testPartitionOSDHelper(t, StoreConfig{StoreType: Filestore})
|
||||
}
|
||||
|
||||
func testPartitionOSDHelper(t *testing.T, storeConfig StoreConfig) {
|
||||
// set up a temporary config directory that will be cleaned up after test
|
||||
configDir, err := ioutil.TempDir("", "TestPartitionBluestoreOSD")
|
||||
if err != nil {
|
||||
@@ -237,17 +243,36 @@ func TestPartitionBluestoreOSD(t *testing.T) {
|
||||
executor := &exectest.MockExecutor{}
|
||||
executor.MockExecuteCommand = func(name string, command string, args ...string) error {
|
||||
logger.Infof("RUN %d for '%s'. %s %+v", execCount, name, command, args)
|
||||
assert.Equal(t, "sgdisk", command)
|
||||
if execCount <= 2 {
|
||||
assert.Equal(t, "sgdisk", command)
|
||||
}
|
||||
switch execCount {
|
||||
case 0:
|
||||
assert.Equal(t, []string{"--zap-all", "/dev/sda"}, args)
|
||||
case 1:
|
||||
assert.Equal(t, []string{"--clear", "--mbrtogpt", "/dev/sda"}, args)
|
||||
case 2:
|
||||
assert.Equal(t, 11, len(args))
|
||||
assert.Equal(t, "--change-name=1:ROOK-OSD1-WAL", args[1])
|
||||
assert.Equal(t, "--change-name=2:ROOK-OSD1-DB", args[4])
|
||||
assert.Equal(t, "--change-name=3:ROOK-OSD1-BLOCK", args[7])
|
||||
if storeConfig.StoreType == Bluestore {
|
||||
assert.Equal(t, 11, len(args))
|
||||
assert.Equal(t, "--change-name=1:ROOK-OSD1-WAL", args[1])
|
||||
assert.Equal(t, "--change-name=2:ROOK-OSD1-DB", args[4])
|
||||
assert.Equal(t, "--change-name=3:ROOK-OSD1-BLOCK", args[7])
|
||||
} else {
|
||||
assert.Equal(t, 5, len(args))
|
||||
assert.Equal(t, "--change-name=1:ROOK-OSD1-FS-DATA", args[1])
|
||||
}
|
||||
case 3:
|
||||
if storeConfig.StoreType == Bluestore {
|
||||
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
|
||||
} else {
|
||||
assert.Equal(t, "mkfs.ext4", command)
|
||||
}
|
||||
case 4:
|
||||
if storeConfig.StoreType == Bluestore {
|
||||
assert.Fail(t, fmt.Sprintf("unexpected case %d", execCount))
|
||||
} else {
|
||||
assert.Equal(t, "mount", command)
|
||||
}
|
||||
}
|
||||
execCount++
|
||||
return nil
|
||||
@@ -260,28 +285,32 @@ func TestPartitionBluestoreOSD(t *testing.T) {
|
||||
}
|
||||
|
||||
// setup a partition scheme for data and metadata to be collocated on sda
|
||||
bluestoreConfig := partition.BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
|
||||
entry := partition.NewPerfSchemeEntry()
|
||||
entry := NewPerfSchemeEntry(storeConfig.StoreType)
|
||||
entry.ID = 1
|
||||
entry.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
partition.PopulateCollocatedPerfSchemeEntry(entry, "sda", bluestoreConfig)
|
||||
PopulateCollocatedPerfSchemeEntry(entry, "sda", storeConfig)
|
||||
|
||||
config := &osdConfig{configRoot: configDir, rootPath: filepath.Join(configDir, "osd1"), id: entry.ID,
|
||||
uuid: entry.OsdUUID, dir: false, partitionScheme: entry}
|
||||
|
||||
// partition the OSD on sda now
|
||||
err = partitionBluestoreOSD(context, config)
|
||||
err = partitionOSD(context, config)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 3, execCount)
|
||||
|
||||
if storeConfig.StoreType == Bluestore {
|
||||
assert.Equal(t, 3, execCount)
|
||||
} else {
|
||||
assert.Equal(t, 5, execCount)
|
||||
}
|
||||
|
||||
// verify that both the data and metadata have been associated with the device in etcd (since data/metadata are collocated)
|
||||
blockDetails, err := getBlockPartitionDetails(config)
|
||||
dataDetails, err := getDataPartitionDetails(config)
|
||||
assert.Nil(t, err)
|
||||
dataID := etcdClient.GetValue(
|
||||
fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-data", blockDetails.DiskUUID))
|
||||
fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-data", dataDetails.DiskUUID))
|
||||
assert.Equal(t, "1", dataID)
|
||||
|
||||
metadataID := etcdClient.GetValue(
|
||||
fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-metadata", blockDetails.DiskUUID))
|
||||
fmt.Sprintf("/rook/services/ceph/osd/desired/node123/device/%s/osd-id-metadata", dataDetails.DiskUUID))
|
||||
assert.Equal(t, "1", metadataID)
|
||||
}
|
||||
@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package partition
|
||||
package osd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@@ -26,21 +26,31 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
WalPartitionName = "wal"
|
||||
DatabasePartitionName = "db"
|
||||
BlockPartitionName = "block"
|
||||
walPartition = 1
|
||||
databasePartition = 2
|
||||
blockPartition = 3
|
||||
UseRemainingSpace = -1
|
||||
schemeFilename = "partition-scheme"
|
||||
WalDefaultSizeMB = 576
|
||||
DBDefaultSizeMB = 20480
|
||||
Filestore = "filestore"
|
||||
Bluestore = "bluestore"
|
||||
DefaultStore = Filestore
|
||||
UseRemainingSpace = -1
|
||||
schemeFilename = "partition-scheme"
|
||||
WalDefaultSizeMB = 576
|
||||
DBDefaultSizeMB = 20480
|
||||
JournalDefaultSizeMB = 5120
|
||||
)
|
||||
|
||||
type BluestoreConfig struct {
|
||||
WalSizeMB int
|
||||
DatabaseSizeMB int
|
||||
type PartitionType int
|
||||
|
||||
const (
|
||||
WalPartitionType PartitionType = iota
|
||||
DatabasePartitionType
|
||||
BlockPartitionType
|
||||
FilestoreDataPartitionType
|
||||
FilestoreJournalPartitionType
|
||||
)
|
||||
|
||||
type StoreConfig struct {
|
||||
StoreType string `json:"storeType,omitempty"`
|
||||
WalSizeMB int `json:"walSizeMB,omitempty"`
|
||||
DatabaseSizeMB int `json:"databaseSizeMB,omitempty"`
|
||||
JournalSizeMB int `json:"journalSizeMB,omitempty"`
|
||||
}
|
||||
|
||||
// top level representation of an overall performance oriented partition scheme, with a dedicated metadata device
|
||||
@@ -52,9 +62,10 @@ type PerfScheme struct {
|
||||
|
||||
// represents an OSD and details about all of its partitions
|
||||
type PerfSchemeEntry struct {
|
||||
ID int `json:"id"`
|
||||
OsdUUID uuid.UUID `json:"osdUuid"`
|
||||
Partitions map[string]*PerfSchemePartitionDetails `json:"partitions"` // mapping of partition name to its details
|
||||
ID int `json:"id"`
|
||||
OsdUUID uuid.UUID `json:"osdUuid"`
|
||||
Partitions map[PartitionType]*PerfSchemePartitionDetails `json:"partitions"` // mapping of partition name to its details
|
||||
StoreType string `json:"storeType,omitempty"`
|
||||
}
|
||||
|
||||
// details for 1 OSD partition
|
||||
@@ -75,12 +86,12 @@ type MetadataDeviceInfo struct {
|
||||
|
||||
// representsa specific partition on a metadata device, including details about which OSD it belongs to
|
||||
type MetadataDevicePartition struct {
|
||||
ID int `json:"id"`
|
||||
OsdUUID uuid.UUID `json:"osdUuid"`
|
||||
Name string `json:"name"`
|
||||
PartitionUUID string `json:"partitionUuid"`
|
||||
SizeMB int `json:"sizeMB"`
|
||||
OffsetMB int `json:"offsetMB"`
|
||||
ID int `json:"id"`
|
||||
OsdUUID uuid.UUID `json:"osdUuid"`
|
||||
Type PartitionType `json:"type"`
|
||||
PartitionUUID string `json:"partitionUuid"`
|
||||
SizeMB int `json:"sizeMB"`
|
||||
OffsetMB int `json:"offsetMB"`
|
||||
}
|
||||
|
||||
func NewPerfScheme() *PerfScheme {
|
||||
@@ -89,9 +100,10 @@ func NewPerfScheme() *PerfScheme {
|
||||
}
|
||||
}
|
||||
|
||||
func NewPerfSchemeEntry() *PerfSchemeEntry {
|
||||
func NewPerfSchemeEntry(storeType string) *PerfSchemeEntry {
|
||||
return &PerfSchemeEntry{
|
||||
Partitions: map[string]*PerfSchemePartitionDetails{}, // mapping of partition name (e.g. WAL) to it's details
|
||||
Partitions: map[PartitionType]*PerfSchemePartitionDetails{}, // mapping of partition name (e.g. WAL) to it's details
|
||||
StoreType: storeType,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,48 +149,66 @@ func (s *PerfScheme) Save(configDir string) error {
|
||||
}
|
||||
|
||||
// populates a partition scheme entry for an OSD where all its partitions are collocated on a single device
|
||||
func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, bluestoreConfig BluestoreConfig) error {
|
||||
diskUUID, walUUID, dbUUID, blockUUID, err := createUUIDs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, storeConfig StoreConfig) error {
|
||||
|
||||
walSize := WalDefaultSizeMB
|
||||
if bluestoreConfig.WalSizeMB > 0 {
|
||||
walSize = bluestoreConfig.WalSizeMB
|
||||
}
|
||||
dbSize := DBDefaultSizeMB
|
||||
if bluestoreConfig.DatabaseSizeMB > 0 {
|
||||
dbSize = bluestoreConfig.DatabaseSizeMB
|
||||
}
|
||||
if storeConfig.StoreType == Filestore {
|
||||
diskUUID, dataUUID, _, err := createFilestoreUUIDs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
offset := 1
|
||||
// the filestore data partition will take up the entire given device (and we do not create a separate partition/entry
|
||||
// for the journal)
|
||||
entry.Partitions[FilestoreDataPartitionType] = &PerfSchemePartitionDetails{
|
||||
Device: device,
|
||||
DiskUUID: diskUUID.String(),
|
||||
PartitionUUID: dataUUID.String(),
|
||||
SizeMB: UseRemainingSpace,
|
||||
OffsetMB: 1,
|
||||
}
|
||||
} else {
|
||||
diskUUID, walUUID, dbUUID, blockUUID, err := createBluestoreUUIDs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// layout the partitions for WAL, DB, and Block
|
||||
entry.Partitions[WalPartitionName] = &PerfSchemePartitionDetails{
|
||||
Device: device,
|
||||
DiskUUID: diskUUID.String(),
|
||||
PartitionUUID: walUUID.String(),
|
||||
SizeMB: walSize,
|
||||
OffsetMB: offset,
|
||||
}
|
||||
offset += entry.Partitions[WalPartitionName].SizeMB
|
||||
walSize := WalDefaultSizeMB
|
||||
if storeConfig.WalSizeMB > 0 {
|
||||
walSize = storeConfig.WalSizeMB
|
||||
}
|
||||
dbSize := DBDefaultSizeMB
|
||||
if storeConfig.DatabaseSizeMB > 0 {
|
||||
dbSize = storeConfig.DatabaseSizeMB
|
||||
}
|
||||
|
||||
entry.Partitions[DatabasePartitionName] = &PerfSchemePartitionDetails{
|
||||
Device: device,
|
||||
DiskUUID: diskUUID.String(),
|
||||
PartitionUUID: dbUUID.String(),
|
||||
SizeMB: dbSize,
|
||||
OffsetMB: offset,
|
||||
}
|
||||
offset += entry.Partitions[DatabasePartitionName].SizeMB
|
||||
offset := 1
|
||||
|
||||
entry.Partitions[BlockPartitionName] = &PerfSchemePartitionDetails{
|
||||
Device: device,
|
||||
DiskUUID: diskUUID.String(),
|
||||
PartitionUUID: blockUUID.String(),
|
||||
SizeMB: UseRemainingSpace,
|
||||
OffsetMB: offset,
|
||||
// layout the partitions for WAL, DB, and Block
|
||||
entry.Partitions[WalPartitionType] = &PerfSchemePartitionDetails{
|
||||
Device: device,
|
||||
DiskUUID: diskUUID.String(),
|
||||
PartitionUUID: walUUID.String(),
|
||||
SizeMB: walSize,
|
||||
OffsetMB: offset,
|
||||
}
|
||||
offset += entry.Partitions[WalPartitionType].SizeMB
|
||||
|
||||
entry.Partitions[DatabasePartitionType] = &PerfSchemePartitionDetails{
|
||||
Device: device,
|
||||
DiskUUID: diskUUID.String(),
|
||||
PartitionUUID: dbUUID.String(),
|
||||
SizeMB: dbSize,
|
||||
OffsetMB: offset,
|
||||
}
|
||||
offset += entry.Partitions[DatabasePartitionType].SizeMB
|
||||
|
||||
entry.Partitions[BlockPartitionType] = &PerfSchemePartitionDetails{
|
||||
Device: device,
|
||||
DiskUUID: diskUUID.String(),
|
||||
PartitionUUID: blockUUID.String(),
|
||||
SizeMB: UseRemainingSpace,
|
||||
OffsetMB: offset,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -187,15 +217,20 @@ func PopulateCollocatedPerfSchemeEntry(entry *PerfSchemeEntry, device string, bl
|
||||
// populates a partition scheme entry for an OSD that will have distributed partitions: its metadata will live on a
|
||||
// dedicated metadata device and its block data will live on a dedicated device
|
||||
func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, metadataInfo *MetadataDeviceInfo,
|
||||
bluestoreConfig BluestoreConfig) error {
|
||||
storeConfig StoreConfig) error {
|
||||
|
||||
diskUUID, walUUID, dbUUID, blockUUID, err := createUUIDs()
|
||||
if storeConfig.StoreType == Filestore {
|
||||
// TODO: support separate metadata device for filestore
|
||||
return fmt.Errorf("filestore not yet supported for distributed partition scheme")
|
||||
}
|
||||
|
||||
diskUUID, walUUID, dbUUID, blockUUID, err := createBluestoreUUIDs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// the block partition will take up the entire given device
|
||||
entry.Partitions[BlockPartitionName] = &PerfSchemePartitionDetails{
|
||||
entry.Partitions[BlockPartitionType] = &PerfSchemePartitionDetails{
|
||||
Device: device,
|
||||
DiskUUID: diskUUID.String(),
|
||||
PartitionUUID: blockUUID.String(),
|
||||
@@ -219,16 +254,16 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m
|
||||
}
|
||||
|
||||
walSize := WalDefaultSizeMB
|
||||
if bluestoreConfig.WalSizeMB > 0 {
|
||||
walSize = bluestoreConfig.WalSizeMB
|
||||
if storeConfig.WalSizeMB > 0 {
|
||||
walSize = storeConfig.WalSizeMB
|
||||
}
|
||||
dbSize := DBDefaultSizeMB
|
||||
if bluestoreConfig.DatabaseSizeMB > 0 {
|
||||
dbSize = bluestoreConfig.DatabaseSizeMB
|
||||
if storeConfig.DatabaseSizeMB > 0 {
|
||||
dbSize = storeConfig.DatabaseSizeMB
|
||||
}
|
||||
|
||||
// record information about the WAL partition
|
||||
entry.Partitions[WalPartitionName] = &PerfSchemePartitionDetails{
|
||||
entry.Partitions[WalPartitionType] = &PerfSchemePartitionDetails{
|
||||
Device: metadataInfo.Device,
|
||||
DiskUUID: metadataInfo.DiskUUID,
|
||||
PartitionUUID: walUUID.String(),
|
||||
@@ -238,16 +273,16 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m
|
||||
walPartitionInfo := &MetadataDevicePartition{
|
||||
ID: entry.ID,
|
||||
OsdUUID: entry.OsdUUID,
|
||||
Name: WalPartitionName,
|
||||
Type: WalPartitionType,
|
||||
PartitionUUID: walUUID.String(),
|
||||
SizeMB: walSize,
|
||||
OffsetMB: offset,
|
||||
}
|
||||
metadataInfo.Partitions = append(metadataInfo.Partitions, walPartitionInfo)
|
||||
offset += entry.Partitions[WalPartitionName].SizeMB
|
||||
offset += entry.Partitions[WalPartitionType].SizeMB
|
||||
|
||||
// record information about the DB partition
|
||||
entry.Partitions[DatabasePartitionName] = &PerfSchemePartitionDetails{
|
||||
entry.Partitions[DatabasePartitionType] = &PerfSchemePartitionDetails{
|
||||
Device: metadataInfo.Device,
|
||||
DiskUUID: metadataInfo.DiskUUID,
|
||||
PartitionUUID: dbUUID.String(),
|
||||
@@ -257,7 +292,7 @@ func PopulateDistributedPerfSchemeEntry(entry *PerfSchemeEntry, device string, m
|
||||
dbPartitionInfo := &MetadataDevicePartition{
|
||||
ID: entry.ID,
|
||||
OsdUUID: entry.OsdUUID,
|
||||
Name: DatabasePartitionName,
|
||||
Type: DatabasePartitionType,
|
||||
PartitionUUID: dbUUID.String(),
|
||||
SizeMB: dbSize,
|
||||
OffsetMB: offset,
|
||||
@@ -271,7 +306,7 @@ func (m *MetadataDeviceInfo) GetPartitionArgs() []string {
|
||||
args := []string{}
|
||||
|
||||
for partNum, part := range m.Partitions {
|
||||
partArgs := getPartitionArgs(partNum+1, part.PartitionUUID, part.OffsetMB, part.SizeMB, getPartitionLabel(part.ID, part.Name))
|
||||
partArgs := getPartitionArgs(partNum+1, part.PartitionUUID, part.OffsetMB, part.SizeMB, getPartitionLabel(part.ID, part.Type))
|
||||
args = append(args, partArgs...)
|
||||
}
|
||||
|
||||
@@ -288,26 +323,28 @@ func (e *PerfSchemeEntry) GetPartitionArgs() []string {
|
||||
args := []string{}
|
||||
partNum := 1
|
||||
|
||||
if collocated {
|
||||
if collocated && e.StoreType == Bluestore {
|
||||
// partitions are collocated, create the metadata partitions on the same device
|
||||
walDetails := e.Partitions[WalPartitionName]
|
||||
partArgs := getPartitionArgsFromDetails(partNum, WalPartitionName, e.ID, walDetails)
|
||||
walDetails := e.Partitions[WalPartitionType]
|
||||
partArgs := getPartitionArgsFromDetails(partNum, WalPartitionType, e.ID, walDetails)
|
||||
args = append(args, partArgs...)
|
||||
partNum++
|
||||
|
||||
dbDetails := e.Partitions[DatabasePartitionName]
|
||||
partArgs = getPartitionArgsFromDetails(partNum, DatabasePartitionName, e.ID, dbDetails)
|
||||
dbDetails := e.Partitions[DatabasePartitionType]
|
||||
partArgs = getPartitionArgsFromDetails(partNum, DatabasePartitionType, e.ID, dbDetails)
|
||||
args = append(args, partArgs...)
|
||||
partNum++
|
||||
}
|
||||
|
||||
dataPartitionType := e.getDataPartitionType()
|
||||
|
||||
// always create the data partition
|
||||
blockDetails := e.Partitions[BlockPartitionName]
|
||||
partArgs := getPartitionArgsFromDetails(partNum, BlockPartitionName, e.ID, blockDetails)
|
||||
args = append(args, partArgs...)
|
||||
dataDetails := e.Partitions[dataPartitionType]
|
||||
dataPartArgs := getPartitionArgsFromDetails(partNum, dataPartitionType, e.ID, dataDetails)
|
||||
args = append(args, dataPartArgs...)
|
||||
|
||||
// append args for the whole device
|
||||
args = append(args, []string{fmt.Sprintf("--disk-guid=%s", blockDetails.DiskUUID), "/dev/" + blockDetails.Device}...)
|
||||
args = append(args, []string{fmt.Sprintf("--disk-guid=%s", dataDetails.DiskUUID), "/dev/" + dataDetails.Device}...)
|
||||
|
||||
return args
|
||||
}
|
||||
@@ -329,10 +366,26 @@ func (e *PerfSchemeEntry) IsCollocated() bool {
|
||||
return collocated
|
||||
}
|
||||
|
||||
func (e *PerfSchemeEntry) getDataPartitionType() PartitionType {
|
||||
if e.StoreType == Filestore {
|
||||
return FilestoreDataPartitionType
|
||||
} else {
|
||||
return BlockPartitionType
|
||||
}
|
||||
}
|
||||
|
||||
func (e *PerfSchemeEntry) getMetadataPartitionType() PartitionType {
|
||||
if e.StoreType == Filestore {
|
||||
return FilestoreJournalPartitionType
|
||||
} else {
|
||||
return DatabasePartitionType
|
||||
}
|
||||
}
|
||||
|
||||
// Get the arguments necessary to create an sgdisk partition with the given parameters.
|
||||
// number is the partition number.
|
||||
// The offset and length are in MB. Under the covers this is translated to sectors.
|
||||
// If the length is -1, all remaining space will be assigned to the partition.
|
||||
// If the length is -1, all remaining space will be assigned to the
|
||||
func getPartitionArgs(number int, guid string, offset, length int, label string) []string {
|
||||
const sectorsPerMB = 2048
|
||||
var newPart string
|
||||
@@ -351,24 +404,28 @@ func getPartitionArgs(number int, guid string, offset, length int, label string)
|
||||
}
|
||||
}
|
||||
|
||||
func getPartitionArgsFromDetails(number int, name string, id int, details *PerfSchemePartitionDetails) []string {
|
||||
return getPartitionArgs(number, details.PartitionUUID, details.OffsetMB, details.SizeMB, getPartitionLabel(id, name))
|
||||
func getPartitionArgsFromDetails(number int, partType PartitionType, id int, details *PerfSchemePartitionDetails) []string {
|
||||
return getPartitionArgs(number, details.PartitionUUID, details.OffsetMB, details.SizeMB, getPartitionLabel(id, partType))
|
||||
}
|
||||
|
||||
func getPartitionLabel(id int, partName string) string {
|
||||
switch partName {
|
||||
case WalPartitionName:
|
||||
func getPartitionLabel(id int, partType PartitionType) string {
|
||||
switch partType {
|
||||
case WalPartitionType:
|
||||
return fmt.Sprintf("ROOK-OSD%d-WAL", id)
|
||||
case DatabasePartitionName:
|
||||
case DatabasePartitionType:
|
||||
return fmt.Sprintf("ROOK-OSD%d-DB", id)
|
||||
case BlockPartitionName:
|
||||
case BlockPartitionType:
|
||||
return fmt.Sprintf("ROOK-OSD%d-BLOCK", id)
|
||||
case FilestoreDataPartitionType:
|
||||
return fmt.Sprintf("ROOK-OSD%d-FS-DATA", id)
|
||||
case FilestoreJournalPartitionType:
|
||||
return fmt.Sprintf("ROOK-OSD%d-FS-JOURNAL", id)
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func createUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
|
||||
func createBluestoreUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
|
||||
diskUUID, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, fmt.Errorf("failed to get disk uuid. %+v", err)
|
||||
@@ -391,3 +448,22 @@ func createUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
|
||||
|
||||
return &diskUUID, &walUUID, &dbUUID, &blockUUID, nil
|
||||
}
|
||||
|
||||
func createFilestoreUUIDs() (*uuid.UUID, *uuid.UUID, *uuid.UUID, error) {
|
||||
diskUUID, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("failed to get disk uuid. %+v", err)
|
||||
}
|
||||
|
||||
dataUUID, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("failed to get data uuid. %+v", err)
|
||||
}
|
||||
|
||||
journalUUID, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("failed to get journal uuid. %+v", err)
|
||||
}
|
||||
|
||||
return &diskUUID, &dataUUID, &journalUUID, nil
|
||||
}
|
||||
@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package partition
|
||||
package osd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -43,15 +43,15 @@ func TestSchemeSaveLoad(t *testing.T) {
|
||||
// add some entries to the scheme
|
||||
scheme.Metadata = NewMetadataDeviceInfo("sda")
|
||||
scheme.Metadata.DiskUUID = uuid.Must(uuid.NewRandom()).String()
|
||||
m1 := &MetadataDevicePartition{ID: 1, OsdUUID: uuid.Must(uuid.NewRandom()), Name: "wal",
|
||||
m1 := &MetadataDevicePartition{ID: 1, OsdUUID: uuid.Must(uuid.NewRandom()), Type: WalPartitionType,
|
||||
PartitionUUID: uuid.Must(uuid.NewRandom()).String(), SizeMB: 100, OffsetMB: 1}
|
||||
m2 := &MetadataDevicePartition{ID: 1, OsdUUID: m1.OsdUUID, Name: "db",
|
||||
m2 := &MetadataDevicePartition{ID: 1, OsdUUID: m1.OsdUUID, Type: DatabasePartitionType,
|
||||
PartitionUUID: uuid.Must(uuid.NewRandom()).String(), SizeMB: 200, OffsetMB: 101}
|
||||
scheme.Metadata.Partitions = append(scheme.Metadata.Partitions, []*MetadataDevicePartition{m1, m2}...)
|
||||
|
||||
e1 := &PerfSchemeEntry{ID: 1, OsdUUID: m1.OsdUUID}
|
||||
e1.Partitions = map[string]*PerfSchemePartitionDetails{
|
||||
"block": &PerfSchemePartitionDetails{
|
||||
e1.Partitions = map[PartitionType]*PerfSchemePartitionDetails{
|
||||
BlockPartitionType: &PerfSchemePartitionDetails{
|
||||
Device: "sdb",
|
||||
DiskUUID: uuid.Must(uuid.NewRandom()).String(),
|
||||
PartitionUUID: uuid.Must(uuid.NewRandom()).String(),
|
||||
@@ -71,45 +71,45 @@ func TestSchemeSaveLoad(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPopulateCollocatedPerfSchemeEntry(t *testing.T) {
|
||||
entry := NewPerfSchemeEntry()
|
||||
entry := NewPerfSchemeEntry(Bluestore)
|
||||
entry.ID = 10
|
||||
entry.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
err := PopulateCollocatedPerfSchemeEntry(entry, "sda", BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2})
|
||||
err := PopulateCollocatedPerfSchemeEntry(entry, "sda", StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2})
|
||||
assert.Nil(t, err)
|
||||
|
||||
// verify the populated collocated partition entries
|
||||
assert.Equal(t, 3, len(entry.Partitions))
|
||||
verifyPartitionDetails(t, entry, WalPartitionName, "sda", 1, 1)
|
||||
verifyPartitionDetails(t, entry, DatabasePartitionName, "sda", 2, 2)
|
||||
verifyPartitionDetails(t, entry, BlockPartitionName, "sda", 4, -1)
|
||||
verifyPartitionDetails(t, entry, WalPartitionType, "sda", 1, 1)
|
||||
verifyPartitionDetails(t, entry, DatabasePartitionType, "sda", 2, 2)
|
||||
verifyPartitionDetails(t, entry, BlockPartitionType, "sda", 4, -1)
|
||||
|
||||
}
|
||||
|
||||
func TestPopulateDistributedPerfSchemeEntry(t *testing.T) {
|
||||
metadata := NewMetadataDeviceInfo("sda")
|
||||
|
||||
entry := NewPerfSchemeEntry()
|
||||
entry := NewPerfSchemeEntry(Bluestore)
|
||||
entry.ID = 20
|
||||
entry.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
|
||||
err := PopulateDistributedPerfSchemeEntry(entry, "sdb", metadata, BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2})
|
||||
err := PopulateDistributedPerfSchemeEntry(entry, "sdb", metadata, StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2})
|
||||
assert.Nil(t, err)
|
||||
|
||||
// verify the populated distributed partition entries (metadata partitions should be on metadata device, block
|
||||
// partition should be on its own device)
|
||||
assert.Equal(t, 3, len(entry.Partitions))
|
||||
verifyPartitionDetails(t, entry, WalPartitionName, "sda", 1, 1)
|
||||
verifyPartitionDetails(t, entry, DatabasePartitionName, "sda", 2, 2)
|
||||
verifyPartitionDetails(t, entry, BlockPartitionName, "sdb", 1, -1)
|
||||
verifyPartitionDetails(t, entry, WalPartitionType, "sda", 1, 1)
|
||||
verifyPartitionDetails(t, entry, DatabasePartitionType, "sda", 2, 2)
|
||||
verifyPartitionDetails(t, entry, BlockPartitionType, "sdb", 1, -1)
|
||||
|
||||
// verify that the metadata device info was populated as well
|
||||
assert.Equal(t, 2, len(metadata.Partitions))
|
||||
verifyMetadataDevicePartition(t, metadata, 0, entry.ID, entry.OsdUUID, WalPartitionName, 1, 1)
|
||||
verifyMetadataDevicePartition(t, metadata, 1, entry.ID, entry.OsdUUID, DatabasePartitionName, 2, 2)
|
||||
verifyMetadataDevicePartition(t, metadata, 0, entry.ID, entry.OsdUUID, WalPartitionType, 1, 1)
|
||||
verifyMetadataDevicePartition(t, metadata, 1, entry.ID, entry.OsdUUID, DatabasePartitionType, 2, 2)
|
||||
}
|
||||
|
||||
func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partName, device string, offset, size int64) {
|
||||
part, ok := entry.Partitions[partName]
|
||||
func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partType PartitionType, device string, offset, size int64) {
|
||||
part, ok := entry.Partitions[partType]
|
||||
assert.True(t, ok)
|
||||
assert.NotNil(t, part)
|
||||
assert.Equal(t, device, part.Device)
|
||||
@@ -118,13 +118,13 @@ func verifyPartitionDetails(t *testing.T, entry *PerfSchemeEntry, partName, devi
|
||||
}
|
||||
|
||||
func verifyMetadataDevicePartition(t *testing.T, info *MetadataDeviceInfo, index int, osdID int, osdUUID uuid.UUID,
|
||||
name string, offset, size int64) {
|
||||
partType PartitionType, offset, size int64) {
|
||||
|
||||
part := info.Partitions[index]
|
||||
assert.NotNil(t, part)
|
||||
assert.Equal(t, osdID, part.ID)
|
||||
assert.Equal(t, osdUUID, part.OsdUUID)
|
||||
assert.Equal(t, name, part.Name)
|
||||
assert.Equal(t, partType, part.Type)
|
||||
assert.Equal(t, offset, part.OffsetMB)
|
||||
assert.Equal(t, size, part.SizeMB)
|
||||
}
|
||||
@@ -132,18 +132,18 @@ func verifyMetadataDevicePartition(t *testing.T, info *MetadataDeviceInfo, index
|
||||
func TestMetadataGetPartitionArgs(t *testing.T) {
|
||||
metadata := NewMetadataDeviceInfo("sda")
|
||||
|
||||
e1 := NewPerfSchemeEntry()
|
||||
e1 := NewPerfSchemeEntry(Bluestore)
|
||||
e1.ID = 1
|
||||
e1.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
|
||||
e2 := NewPerfSchemeEntry()
|
||||
e2 := NewPerfSchemeEntry(Bluestore)
|
||||
e2.ID = 2
|
||||
e2.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
|
||||
bluestoreConfig := BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
|
||||
err := PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, bluestoreConfig)
|
||||
storeConfig := StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
|
||||
err := PopulateDistributedPerfSchemeEntry(e1, "sdb", metadata, storeConfig)
|
||||
assert.Nil(t, err)
|
||||
err = PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, bluestoreConfig)
|
||||
err = PopulateDistributedPerfSchemeEntry(e2, "sdc", metadata, storeConfig)
|
||||
assert.Nil(t, err)
|
||||
|
||||
expectedArgs := []string{
|
||||
@@ -160,19 +160,19 @@ func TestMetadataGetPartitionArgs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSchemeEntryGetPartitionArgs(t *testing.T) {
|
||||
e1 := NewPerfSchemeEntry()
|
||||
e1 := NewPerfSchemeEntry(Bluestore)
|
||||
e1.ID = 1
|
||||
e1.OsdUUID = uuid.Must(uuid.NewRandom())
|
||||
|
||||
bluestoreConfig := BluestoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
|
||||
err := PopulateCollocatedPerfSchemeEntry(e1, "sdb", bluestoreConfig)
|
||||
storeConfig := StoreConfig{WalSizeMB: 1, DatabaseSizeMB: 2}
|
||||
err := PopulateCollocatedPerfSchemeEntry(e1, "sdb", storeConfig)
|
||||
assert.Nil(t, err)
|
||||
|
||||
expectedArgs := []string{
|
||||
"--new=1:2048:+2048", "--change-name=1:ROOK-OSD1-WAL", fmt.Sprintf("--partition-guid=1:%s", e1.Partitions[WalPartitionName].PartitionUUID),
|
||||
"--new=2:4096:+4096", "--change-name=2:ROOK-OSD1-DB", fmt.Sprintf("--partition-guid=2:%s", e1.Partitions[DatabasePartitionName].PartitionUUID),
|
||||
"--largest-new=3", "--change-name=3:ROOK-OSD1-BLOCK", fmt.Sprintf("--partition-guid=3:%s", e1.Partitions[BlockPartitionName].PartitionUUID),
|
||||
fmt.Sprintf("--disk-guid=%s", e1.Partitions[BlockPartitionName].DiskUUID), "/dev/sdb",
|
||||
"--new=1:2048:+2048", "--change-name=1:ROOK-OSD1-WAL", fmt.Sprintf("--partition-guid=1:%s", e1.Partitions[WalPartitionType].PartitionUUID),
|
||||
"--new=2:4096:+4096", "--change-name=2:ROOK-OSD1-DB", fmt.Sprintf("--partition-guid=2:%s", e1.Partitions[DatabasePartitionType].PartitionUUID),
|
||||
"--largest-new=3", "--change-name=3:ROOK-OSD1-BLOCK", fmt.Sprintf("--partition-guid=3:%s", e1.Partitions[BlockPartitionType].PartitionUUID),
|
||||
fmt.Sprintf("--disk-guid=%s", e1.Partitions[BlockPartitionType].DiskUUID), "/dev/sdb",
|
||||
}
|
||||
|
||||
// get the partition args and verify against expected
|
||||
@@ -99,8 +99,7 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
|
||||
|
||||
func (c *Cluster) apiContainer() v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd api --data-dir=%s --port=%d --container-version=%s",
|
||||
k8sutil.DataDir, model.Port, c.Version)
|
||||
command := fmt.Sprintf("/usr/bin/rookd api --config-dir=%s --port=%d --container-version=%s", k8sutil.DataDir, model.Port, c.Version)
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
// Without waiting some time, there is highly probable flakes in network setup.
|
||||
|
||||
@@ -78,8 +78,8 @@ func TestPodSpecs(t *testing.T) {
|
||||
for _, v := range cont.Env {
|
||||
assert.True(t, strings.HasPrefix(v.Name, "ROOKD_"))
|
||||
}
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rookd api --data-dir=/var/lib/rook --port=%d --container-version=%s",
|
||||
model.Port, c.Version)
|
||||
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rookd api --config-dir=/var/lib/rook --port=%d --container-version=%s", model.Port, c.Version)
|
||||
|
||||
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
|
||||
}
|
||||
@@ -92,12 +92,12 @@ func (t *clusterTPR) findAllClusters() error {
|
||||
}
|
||||
|
||||
func (t *clusterTPR) startCluster(c *cluster.Cluster) {
|
||||
if t.devicesInUse && c.Spec.UseAllDevices {
|
||||
if t.devicesInUse && c.Spec.Storage.AnyUseAllDevices() {
|
||||
logger.Warningf("devices in more than one namespace not supported. ignoring devices in namespace %s", c.Spec.Namespace)
|
||||
c.Spec.UseAllDevices = false
|
||||
c.Spec.Storage.ClearUseAllDevices()
|
||||
}
|
||||
|
||||
if c.Spec.UseAllDevices {
|
||||
if c.Spec.Storage.AnyUseAllDevices() {
|
||||
t.devicesInUse = true
|
||||
}
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ func (c *Cluster) CreateInstance() error {
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices)
|
||||
c.osds = osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.Storage, c.Spec.DataDirHostPath)
|
||||
err = c.osds.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the osds. %+v", err)
|
||||
|
||||
@@ -18,6 +18,8 @@ which also has the apache 2.0 license.
|
||||
*/
|
||||
package cluster
|
||||
|
||||
import "github.com/rook/rook/pkg/operator/osd"
|
||||
|
||||
type Spec struct {
|
||||
// The namespace where the the rook resources will all be created.
|
||||
Namespace string `json:"namespace"`
|
||||
@@ -30,14 +32,11 @@ type Spec struct {
|
||||
// Paused is to pause the control of the operator for the rook cluster.
|
||||
Paused bool `json:"paused,omitempty"`
|
||||
|
||||
// Whether to consume all the storage devices found on a machine
|
||||
UseAllDevices bool `json:"useAllDevices"`
|
||||
|
||||
// A regular expression to allow more fine-grained selection of devices on nodes across the cluster
|
||||
DeviceFilter string `json:"deviceFilter"`
|
||||
|
||||
// The path on the host where config and data can be persisted.
|
||||
DataDirHostPath string `json:"dataDirHostPath"`
|
||||
|
||||
// A spec for available storage in the cluster and how it should be used
|
||||
Storage osd.StorageSpec `json:"storage"`
|
||||
}
|
||||
|
||||
type PoolSpec struct {
|
||||
|
||||
@@ -46,6 +46,10 @@ func RepoPrefixEnvVar() v1.EnvVar {
|
||||
return v1.EnvVar{Name: repoPrefixEnvVar, Value: repoPrefix()}
|
||||
}
|
||||
|
||||
func ConfigDirEnvVar() v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_CONFIG_DIR", Value: DataDir}
|
||||
}
|
||||
|
||||
func repoPrefix() string {
|
||||
r := os.Getenv(repoPrefixEnvVar)
|
||||
if r == "" {
|
||||
|
||||
@@ -13,8 +13,22 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package partition
|
||||
package k8sutil
|
||||
|
||||
import "github.com/coreos/pkg/capnslog"
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephosdpart")
|
||||
func PathToVolumeName(path string) string {
|
||||
// kubernetes volume names must match this regex: [a-z0-9]([-a-z0-9]*[a-z0-9])?
|
||||
|
||||
// first replace all filepath separators with hyphens
|
||||
volumeName := strings.Replace(path, string(filepath.Separator), "-", -1)
|
||||
|
||||
// trim any leading/trailing hyphens
|
||||
volumeName = strings.TrimPrefix(volumeName, "-")
|
||||
volumeName = strings.TrimSuffix(volumeName, "-")
|
||||
|
||||
return volumeName
|
||||
}
|
||||
@@ -150,7 +150,7 @@ func (c *Cluster) makeDeployment(id string) *extensions.Deployment {
|
||||
|
||||
func (c *Cluster) mdsContainer(id string) v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd mds --data-dir=%s --mds-id=%s ",
|
||||
command := fmt.Sprintf("/usr/bin/rookd mds --config-dir=%s --mds-id=%s ",
|
||||
k8sutil.DataDir, id)
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestPodSpecs(t *testing.T) {
|
||||
assert.Equal(t, 1, len(cont.VolumeMounts))
|
||||
assert.Equal(t, 5, len(cont.Env))
|
||||
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rookd mds --data-dir=/var/lib/rook --mds-id=%s ",
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rookd mds --config-dir=/var/lib/rook --mds-id=%s ",
|
||||
mdsID)
|
||||
|
||||
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
|
||||
|
||||
@@ -83,7 +83,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, antiAffinity bool) *v1.Pod {
|
||||
}
|
||||
|
||||
func (c *Cluster) monContainer(config *MonConfig, fsid string) v1.Container {
|
||||
command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --port=%d --fsid=%s",
|
||||
command := fmt.Sprintf("/usr/bin/rookd mon --config-dir=%s --name=%s --port=%d --fsid=%s",
|
||||
k8sutil.DataDir, config.Name, config.Port, fsid)
|
||||
|
||||
return v1.Container{
|
||||
|
||||
@@ -62,7 +62,7 @@ func testPodSpec(t *testing.T, dataDir string) {
|
||||
assert.Equal(t, 1, len(cont.VolumeMounts))
|
||||
assert.Equal(t, 5, len(cont.Env))
|
||||
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rookd mon --data-dir=/var/lib/rook --name=%s --port=%d --fsid=%s",
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rookd mon --config-dir=/var/lib/rook --name=%s --port=%d --fsid=%s",
|
||||
config.Name, config.Port, c.clusterInfo.FSID)
|
||||
|
||||
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
|
||||
|
||||
+190
-46
@@ -18,16 +18,22 @@ package osd
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"strings"
|
||||
|
||||
"strconv"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
opmon "github.com/rook/rook/pkg/operator/mon"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
|
||||
)
|
||||
|
||||
const (
|
||||
appName = "osd"
|
||||
appName = "osd"
|
||||
appNameFmt = "osd-%s"
|
||||
)
|
||||
|
||||
type Cluster struct {
|
||||
@@ -35,50 +41,112 @@ type Cluster struct {
|
||||
Namespace string
|
||||
Keyring string
|
||||
Version string
|
||||
useAllDevices bool
|
||||
Storage StorageSpec
|
||||
dataDirHostPath string
|
||||
deviceFilter string
|
||||
}
|
||||
|
||||
func New(clientset kubernetes.Interface, namespace, version, deviceFilter, dataDirHostPath string, useAllDevices bool) *Cluster {
|
||||
func New(clientset kubernetes.Interface, namespace, version string, storageSpec StorageSpec, dataDirHostPath string) *Cluster {
|
||||
return &Cluster{
|
||||
clientset: clientset,
|
||||
Namespace: namespace,
|
||||
Version: version,
|
||||
deviceFilter: deviceFilter,
|
||||
Storage: storageSpec,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
useAllDevices: useAllDevices,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) Start() error {
|
||||
logger.Infof("start running osds")
|
||||
logger.Infof("start running osds in namespace %s", c.Namespace)
|
||||
|
||||
ds, err := c.makeDaemonSet()
|
||||
_, err = c.clientset.Extensions().DaemonSets(c.Namespace).Create(ds)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create osd daemon set. %+v", err)
|
||||
if c.Storage.UseAllNodes {
|
||||
// make a daemonset for all nodes in the cluster
|
||||
ds := c.makeDaemonSet(c.Storage.Selection, c.Storage.Config)
|
||||
_, err := c.clientset.Extensions().DaemonSets(c.Namespace).Create(ds)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create osd daemon set. %+v", err)
|
||||
}
|
||||
logger.Infof("osd daemon set already exists")
|
||||
} else {
|
||||
logger.Infof("osd daemon set started")
|
||||
}
|
||||
logger.Infof("osd daemon set already exists")
|
||||
} else {
|
||||
logger.Infof("osd daemon set started")
|
||||
for i := range c.Storage.Nodes {
|
||||
// fully resolve the storage config for this node
|
||||
n := c.Storage.resolveNode(c.Storage.Nodes[i].Name)
|
||||
|
||||
// create the replicaSet that will run the OSDs for this node
|
||||
rs := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
|
||||
_, err := c.clientset.Extensions().ReplicaSets(c.Namespace).Create(rs)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create osd replica set for node %s. %+v", n.Name, err)
|
||||
}
|
||||
logger.Infof("osd replica set already exists for node %s", n.Name)
|
||||
} else {
|
||||
logger.Infof("osd replica set started for node %s", n.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDaemonSet() (*extensions.DaemonSet, error) {
|
||||
func (c *Cluster) makeDaemonSet(selection Selection, config Config) *extensions.DaemonSet {
|
||||
ds := &extensions.DaemonSet{}
|
||||
ds.Name = appName
|
||||
ds.Namespace = c.Namespace
|
||||
|
||||
podSpec := c.podTemplateSpec(nil, nil, selection, config)
|
||||
|
||||
ds.Spec = extensions.DaemonSetSpec{Template: podSpec}
|
||||
return ds
|
||||
}
|
||||
|
||||
func (c *Cluster) makeReplicaSet(nodeName string, devices []Device, directories []Directory,
|
||||
selection Selection, config Config) *extensions.ReplicaSet {
|
||||
|
||||
rs := &extensions.ReplicaSet{}
|
||||
rs.Name = fmt.Sprintf(appNameFmt, nodeName)
|
||||
rs.Namespace = c.Namespace
|
||||
|
||||
podSpec := c.podTemplateSpec(devices, directories, selection, config)
|
||||
podSpec.Spec.NodeSelector = map[string]string{unversioned.LabelHostname: nodeName}
|
||||
|
||||
replicaCount := int32(1)
|
||||
|
||||
rs.Spec = extensions.ReplicaSetSpec{
|
||||
Template: podSpec,
|
||||
Replicas: &replicaCount,
|
||||
}
|
||||
|
||||
return rs
|
||||
}
|
||||
|
||||
func (c *Cluster) podTemplateSpec(devices []Device, directories []Directory, selection Selection, config Config) v1.PodTemplateSpec {
|
||||
// by default, the data/config dir will be an empty volume
|
||||
dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}
|
||||
if c.dataDirHostPath != "" {
|
||||
// the user has specified a host path to use for the data dir, use that instead
|
||||
dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
|
||||
}
|
||||
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
// create volume config for the data dir and /dev so the pod can access devices on the host
|
||||
volumes := []v1.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource},
|
||||
{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}},
|
||||
}
|
||||
|
||||
// add each OSD directory as another host path volume source
|
||||
for _, d := range directories {
|
||||
dirVolume := v1.Volume{
|
||||
Name: k8sutil.PathToVolumeName(d.Path),
|
||||
VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: d.Path}},
|
||||
}
|
||||
volumes = append(volumes, dirVolume)
|
||||
}
|
||||
|
||||
return v1.PodTemplateSpec{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: appName,
|
||||
Labels: map[string]string{
|
||||
@@ -88,48 +156,124 @@ func (c *Cluster) makeDaemonSet() (*extensions.DaemonSet, error) {
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.osdContainer()},
|
||||
Containers: []v1.Container{c.osdContainer(devices, directories, selection, config)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: []v1.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource},
|
||||
{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}},
|
||||
},
|
||||
Volumes: volumes,
|
||||
},
|
||||
}
|
||||
|
||||
ds.Spec = extensions.DaemonSetSpec{Template: podSpec}
|
||||
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) osdContainer() v1.Container {
|
||||
func (c *Cluster) osdContainer(devices []Device, directories []Directory, selection Selection, config Config) v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s ", k8sutil.DataDir)
|
||||
var devices string
|
||||
if c.deviceFilter != "" {
|
||||
devices = c.deviceFilter
|
||||
} else if c.useAllDevices {
|
||||
devices = "all"
|
||||
command := "/usr/bin/rookd osd"
|
||||
|
||||
envVars := []v1.EnvVar{
|
||||
opmon.ClusterNameEnvVar(),
|
||||
opmon.MonEndpointEnvVar(),
|
||||
opmon.MonSecretEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
k8sutil.ConfigDirEnvVar(),
|
||||
}
|
||||
|
||||
// only 1 of device list, device filter and use all devices can be specified. We prioritize in that order.
|
||||
if len(devices) > 0 {
|
||||
deviceNames := make([]string, len(devices))
|
||||
for i := range devices {
|
||||
deviceNames[i] = devices[i].Name
|
||||
}
|
||||
envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ",")))
|
||||
} else if selection.DeviceFilter != "" {
|
||||
envVars = append(envVars, deviceFilterEnvVar(selection.DeviceFilter))
|
||||
} else if selection.getUseAllDevices() {
|
||||
envVars = append(envVars, deviceFilterEnvVar("all"))
|
||||
}
|
||||
|
||||
if selection.MetadataDevice != "" {
|
||||
envVars = append(envVars, metadataDeviceEnvVar(selection.MetadataDevice))
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
{Name: "devices", MountPath: "/dev"},
|
||||
}
|
||||
|
||||
if len(directories) > 0 {
|
||||
// for each directory the user has specified, create a volume mount and pass it to the pod via cmd line arg
|
||||
dirPaths := make([]string, len(directories))
|
||||
for i := range directories {
|
||||
dpath := directories[i].Path
|
||||
dirPaths[i] = dpath
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: k8sutil.PathToVolumeName(dpath), MountPath: dpath})
|
||||
}
|
||||
|
||||
envVars = append(envVars, dataDirectoriesEnvVar(strings.Join(dirPaths, ",")))
|
||||
}
|
||||
|
||||
if config.StoreConfig.StoreType != "" {
|
||||
envVars = append(envVars, osdStoreEnvVar(config.StoreConfig.StoreType))
|
||||
}
|
||||
|
||||
if config.StoreConfig.DatabaseSizeMB != 0 {
|
||||
envVars = append(envVars, osdDatabaseSizeEnvVar(config.StoreConfig.DatabaseSizeMB))
|
||||
}
|
||||
|
||||
if config.StoreConfig.WalSizeMB != 0 {
|
||||
envVars = append(envVars, osdWalSizeEnvVar(config.StoreConfig.WalSizeMB))
|
||||
}
|
||||
|
||||
if config.StoreConfig.JournalSizeMB != 0 {
|
||||
envVars = append(envVars, osdJournalSizeEnvVar(config.StoreConfig.JournalSizeMB))
|
||||
}
|
||||
|
||||
if config.Location != "" {
|
||||
envVars = append(envVars, locationEnvVar(config.Location))
|
||||
}
|
||||
|
||||
privileged := true
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
// Without waiting some time, there is highly probable flakes in network setup.
|
||||
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
|
||||
Name: appName,
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
{Name: "devices", MountPath: "/dev"},
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
v1.EnvVar{Name: "ROOKD_DATA_DEVICES", Value: devices},
|
||||
opmon.ClusterNameEnvVar(),
|
||||
opmon.MonEndpointEnvVar(),
|
||||
opmon.MonSecretEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
},
|
||||
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
|
||||
Name: appName,
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
VolumeMounts: volumeMounts,
|
||||
Env: envVars,
|
||||
SecurityContext: &v1.SecurityContext{Privileged: &privileged},
|
||||
}
|
||||
}
|
||||
|
||||
func dataDevicesEnvVar(dataDevices string) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_DATA_DEVICES", Value: dataDevices}
|
||||
}
|
||||
|
||||
func deviceFilterEnvVar(filter string) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_DATA_DEVICE_FILTER", Value: filter}
|
||||
}
|
||||
|
||||
func metadataDeviceEnvVar(metadataDevice string) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_METADATA_DEVICE", Value: metadataDevice}
|
||||
}
|
||||
|
||||
func dataDirectoriesEnvVar(dataDirectories string) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_DATA_DIRECTORIES", Value: dataDirectories}
|
||||
}
|
||||
|
||||
func osdStoreEnvVar(osdStore string) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_OSD_STORE", Value: osdStore}
|
||||
}
|
||||
|
||||
func osdDatabaseSizeEnvVar(databaseSize int) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_OSD_DATABASE_SIZE", Value: strconv.Itoa(databaseSize)}
|
||||
}
|
||||
|
||||
func osdWalSizeEnvVar(walSize int) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_OSD_WAL_SIZE", Value: strconv.Itoa(walSize)}
|
||||
}
|
||||
|
||||
func osdJournalSizeEnvVar(journalSize int) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_OSD_JOURNAL_SIZE", Value: strconv.Itoa(journalSize)}
|
||||
}
|
||||
|
||||
func locationEnvVar(location string) v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_LOCATION", Value: location}
|
||||
}
|
||||
+130
-30
@@ -16,6 +16,7 @@ limitations under the License.
|
||||
package osd
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -23,11 +24,14 @@ import (
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/client-go/pkg/api/unversioned"
|
||||
|
||||
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
|
||||
)
|
||||
|
||||
func TestStartDaemonset(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(clientset, "ns", "myversion", "", "", false)
|
||||
c := New(clientset, "ns", "myversion", StorageSpec{}, "")
|
||||
|
||||
// Start the first time
|
||||
err := c.Start()
|
||||
@@ -46,44 +50,140 @@ func TestDaemonset(t *testing.T) {
|
||||
}
|
||||
|
||||
func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(clientset, "ns", "myversion", deviceFilter, dataDir, allDevices)
|
||||
|
||||
daemonSet, err := c.makeDaemonSet()
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, daemonSet)
|
||||
assert.Equal(t, "osd", daemonSet.Name)
|
||||
assert.Equal(t, c.Namespace, daemonSet.Namespace)
|
||||
assert.Equal(t, v1.RestartPolicyAlways, daemonSet.Spec.Template.Spec.RestartPolicy)
|
||||
assert.Equal(t, 2, len(daemonSet.Spec.Template.Spec.Volumes))
|
||||
assert.Equal(t, "rook-data", daemonSet.Spec.Template.Spec.Volumes[0].Name)
|
||||
assert.Equal(t, "devices", daemonSet.Spec.Template.Spec.Volumes[1].Name)
|
||||
if dataDir == "" {
|
||||
assert.NotNil(t, daemonSet.Spec.Template.Spec.Volumes[0].EmptyDir)
|
||||
assert.Nil(t, daemonSet.Spec.Template.Spec.Volumes[0].HostPath)
|
||||
} else {
|
||||
assert.Nil(t, daemonSet.Spec.Template.Spec.Volumes[0].EmptyDir)
|
||||
assert.Equal(t, dataDir, daemonSet.Spec.Template.Spec.Volumes[0].HostPath.Path)
|
||||
storageSpec := StorageSpec{
|
||||
Selection: Selection{UseAllDevices: &allDevices, DeviceFilter: deviceFilter},
|
||||
Nodes: []Node{{Name: "node1"}},
|
||||
}
|
||||
|
||||
assert.Equal(t, "osd", daemonSet.Spec.Template.ObjectMeta.Name)
|
||||
assert.Equal(t, "osd", daemonSet.Spec.Template.ObjectMeta.Labels["app"])
|
||||
assert.Equal(t, c.Namespace, daemonSet.Spec.Template.ObjectMeta.Labels["rook_cluster"])
|
||||
assert.Equal(t, 0, len(daemonSet.Spec.Template.ObjectMeta.Annotations))
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(clientset, "ns", "myversion", storageSpec, dataDir)
|
||||
|
||||
cont := daemonSet.Spec.Template.Spec.Containers[0]
|
||||
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
|
||||
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
|
||||
assert.NotNil(t, replicaSet)
|
||||
assert.Equal(t, "osd-node1", replicaSet.Name)
|
||||
assert.Equal(t, c.Namespace, replicaSet.Namespace)
|
||||
assert.Equal(t, int32(1), *(replicaSet.Spec.Replicas))
|
||||
assert.Equal(t, "node1", replicaSet.Spec.Template.Spec.NodeSelector[unversioned.LabelHostname])
|
||||
assert.Equal(t, v1.RestartPolicyAlways, replicaSet.Spec.Template.Spec.RestartPolicy)
|
||||
assert.Equal(t, 2, len(replicaSet.Spec.Template.Spec.Volumes))
|
||||
assert.Equal(t, "rook-data", replicaSet.Spec.Template.Spec.Volumes[0].Name)
|
||||
assert.Equal(t, "devices", replicaSet.Spec.Template.Spec.Volumes[1].Name)
|
||||
if dataDir == "" {
|
||||
assert.NotNil(t, replicaSet.Spec.Template.Spec.Volumes[0].EmptyDir)
|
||||
assert.Nil(t, replicaSet.Spec.Template.Spec.Volumes[0].HostPath)
|
||||
} else {
|
||||
assert.Nil(t, replicaSet.Spec.Template.Spec.Volumes[0].EmptyDir)
|
||||
assert.Equal(t, dataDir, replicaSet.Spec.Template.Spec.Volumes[0].HostPath.Path)
|
||||
}
|
||||
|
||||
assert.Equal(t, "osd", replicaSet.Spec.Template.ObjectMeta.Name)
|
||||
assert.Equal(t, "osd", replicaSet.Spec.Template.ObjectMeta.Labels["app"])
|
||||
assert.Equal(t, c.Namespace, replicaSet.Spec.Template.ObjectMeta.Labels["rook_cluster"])
|
||||
assert.Equal(t, 0, len(replicaSet.Spec.Template.ObjectMeta.Annotations))
|
||||
|
||||
cont := replicaSet.Spec.Template.Spec.Containers[0]
|
||||
assert.Equal(t, "quay.io/rook/rookd:myversion", cont.Image)
|
||||
assert.Equal(t, 2, len(cont.VolumeMounts))
|
||||
assert.Equal(t, 5, len(cont.Env))
|
||||
|
||||
expectedCommand := "/usr/bin/rookd osd --data-dir=/var/lib/rook "
|
||||
expectedCommand := "/usr/bin/rookd osd"
|
||||
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
|
||||
devices := cont.Env[0].Value
|
||||
|
||||
// verify the config dir env var
|
||||
verifyEnvVar(t, cont.Env, "ROOKD_CONFIG_DIR", "/var/lib/rook", true)
|
||||
|
||||
// verify the osd store type env var uses the default
|
||||
verifyEnvVar(t, cont.Env, "ROOKD_OSD_STORE", cephosd.DefaultStore, true)
|
||||
|
||||
// verify the device filter env var
|
||||
if deviceFilter != "" {
|
||||
assert.Equal(t, deviceFilter, devices)
|
||||
verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", deviceFilter, true)
|
||||
} else if allDevices {
|
||||
assert.Equal(t, "all", devices)
|
||||
verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", "all", true)
|
||||
} else {
|
||||
assert.Equal(t, "", devices)
|
||||
verifyEnvVar(t, cont.Env, "ROOKD_DATA_DEVICE_FILTER", "", false)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyEnvVar(t *testing.T, envVars []v1.EnvVar, expectedName, expectedValue string, expectedFound bool) {
|
||||
found := false
|
||||
for _, envVar := range envVars {
|
||||
if envVar.Name == expectedName {
|
||||
assert.Equal(t, expectedValue, envVar.Value)
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, expectedFound, found)
|
||||
}
|
||||
|
||||
func TestStorageSpecDevicesAndDirectories(t *testing.T) {
|
||||
storageSpec := StorageSpec{
|
||||
Config: Config{},
|
||||
Nodes: []Node{
|
||||
{
|
||||
Name: "node1",
|
||||
Devices: []Device{{Name: "sda"}},
|
||||
Directories: []Directory{{Path: "/rook/dir1"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(clientset, "ns", "myversion", storageSpec, "")
|
||||
|
||||
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
|
||||
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
|
||||
assert.NotNil(t, replicaSet)
|
||||
|
||||
// pod spec should have a volume for the given dir
|
||||
podSpec := replicaSet.Spec.Template.Spec
|
||||
assert.Equal(t, 3, len(podSpec.Volumes))
|
||||
assert.Equal(t, "rook-dir1", podSpec.Volumes[2].Name)
|
||||
assert.Equal(t, "/rook/dir1", podSpec.Volumes[2].VolumeSource.HostPath.Path)
|
||||
|
||||
// container should have a volume mount for the given dir
|
||||
container := podSpec.Containers[0]
|
||||
assert.Equal(t, "rook-dir1", container.VolumeMounts[2].Name)
|
||||
assert.Equal(t, "/rook/dir1", container.VolumeMounts[2].MountPath)
|
||||
|
||||
// container command should have the given dir and device
|
||||
verifyEnvVar(t, container.Env, "ROOKD_DATA_DIRECTORIES", "/rook/dir1", true)
|
||||
verifyEnvVar(t, container.Env, "ROOKD_DATA_DEVICES", "sda", true)
|
||||
}
|
||||
|
||||
func TestStorageSpecConfig(t *testing.T) {
|
||||
storageSpec := StorageSpec{
|
||||
Config: Config{},
|
||||
Nodes: []Node{
|
||||
{
|
||||
Name: "node1",
|
||||
Config: Config{
|
||||
Location: "rack=foo",
|
||||
StoreConfig: cephosd.StoreConfig{
|
||||
StoreType: cephosd.Bluestore,
|
||||
DatabaseSizeMB: 10,
|
||||
WalSizeMB: 20,
|
||||
JournalSizeMB: 30,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(clientset, "ns", "myversion", storageSpec, "")
|
||||
|
||||
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
|
||||
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
|
||||
assert.NotNil(t, replicaSet)
|
||||
|
||||
container := replicaSet.Spec.Template.Spec.Containers[0]
|
||||
assert.NotNil(t, container)
|
||||
verifyEnvVar(t, container.Env, "ROOKD_OSD_STORE", cephosd.Bluestore, true)
|
||||
verifyEnvVar(t, container.Env, "ROOKD_OSD_DATABASE_SIZE", strconv.Itoa(10), true)
|
||||
verifyEnvVar(t, container.Env, "ROOKD_OSD_WAL_SIZE", strconv.Itoa(20), true)
|
||||
verifyEnvVar(t, container.Env, "ROOKD_OSD_JOURNAL_SIZE", strconv.Itoa(30), true)
|
||||
verifyEnvVar(t, container.Env, "ROOKD_LOCATION", "rack=foo", true)
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package osd
|
||||
|
||||
import (
|
||||
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
|
||||
)
|
||||
|
||||
type StorageSpec struct {
|
||||
Nodes []Node `json:"nodes,omitempty"`
|
||||
UseAllNodes bool `json:"useAllNodes,omitempty"`
|
||||
Selection
|
||||
Config
|
||||
}
|
||||
|
||||
type Node struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Devices []Device `json:"devices,omitempty"`
|
||||
Directories []Directory `json:"directories,omitempty"`
|
||||
Selection
|
||||
Config
|
||||
}
|
||||
|
||||
type Device struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
type Directory struct {
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
type Selection struct {
|
||||
// Whether to consume all the storage devices found on a machine
|
||||
UseAllDevices *bool `json:"useAllDevices,omitempty"`
|
||||
|
||||
// A regular expression to allow more fine-grained selection of devices on nodes across the cluster
|
||||
DeviceFilter string `json:"deviceFilter,omitempty"`
|
||||
|
||||
MetadataDevice string `json:"metadataDevice,omitempty"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
StoreConfig cephosd.StoreConfig `json:"storeConfig,omitempty"`
|
||||
Location string `json:"location,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package osd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
|
||||
)
|
||||
|
||||
func TestStorageSpecMarshal(t *testing.T) {
|
||||
specYaml := []byte(`useAllNodes: true
|
||||
useAllDevices: false
|
||||
deviceFilter: "^nvme."
|
||||
metadataDevice:
|
||||
location: "region=us-west,datacenter=delmar"
|
||||
storeConfig:
|
||||
storeType: bluestore
|
||||
journalSizeMB: 1024
|
||||
databaseSizeMB: 1024
|
||||
nodes:
|
||||
- name: "node1"
|
||||
storeConfig:
|
||||
storeType: filestore
|
||||
directories:
|
||||
- path: "/rook/dir1"
|
||||
- name: "node2"
|
||||
deviceFilter: "^sd."`)
|
||||
|
||||
// convert the raw spec yaml into JSON
|
||||
rawJson, err := yaml.YAMLToJSON(specYaml)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// unmarshal the JSON into a strongly typed storage spec object
|
||||
var storageSpec StorageSpec
|
||||
err = json.Unmarshal(rawJson, &storageSpec)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// the unmarshalled storage spec should equal the expected spec below
|
||||
expectedSpec := StorageSpec{
|
||||
UseAllNodes: true,
|
||||
Selection: Selection{
|
||||
UseAllDevices: newBool(false),
|
||||
DeviceFilter: "^nvme.",
|
||||
},
|
||||
Config: Config{
|
||||
Location: "region=us-west,datacenter=delmar",
|
||||
StoreConfig: cephosd.StoreConfig{
|
||||
DatabaseSizeMB: 1024,
|
||||
JournalSizeMB: 1024,
|
||||
StoreType: cephosd.Bluestore,
|
||||
},
|
||||
},
|
||||
Nodes: []Node{
|
||||
Node{
|
||||
Name: "node1",
|
||||
Directories: []Directory{{Path: "/rook/dir1"}},
|
||||
Config: Config{
|
||||
StoreConfig: cephosd.StoreConfig{
|
||||
StoreType: cephosd.Filestore,
|
||||
},
|
||||
},
|
||||
},
|
||||
Node{
|
||||
Name: "node2",
|
||||
Selection: Selection{
|
||||
DeviceFilter: "^sd.",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert.Equal(t, expectedSpec, storageSpec)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package osd
|
||||
|
||||
import (
|
||||
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
|
||||
)
|
||||
|
||||
func (s *StorageSpec) AnyUseAllDevices() bool {
|
||||
if s.Selection.getUseAllDevices() {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, n := range s.Nodes {
|
||||
if n.Selection.getUseAllDevices() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *StorageSpec) ClearUseAllDevices() {
|
||||
clear := false
|
||||
s.Selection.UseAllDevices = &clear
|
||||
for i := range s.Nodes {
|
||||
s.Nodes[i].Selection.UseAllDevices = &clear
|
||||
}
|
||||
}
|
||||
|
||||
// Fully resolves the config of the given node name, taking into account cluster level and node level specified config.
|
||||
// In general, the more fine grained the configuration is specified, the more precedence it takes. Fully resolved
|
||||
// configuration for the node has the following order of precedence.
|
||||
// 1) Node (config defined on the node itself)
|
||||
// 2) Cluster (config defined on the cluster)
|
||||
// 3) Default values (if no config exists for the node or cluster)
|
||||
func (s *StorageSpec) resolveNode(nodeName string) *Node {
|
||||
// find the requested storage node first, if it exists
|
||||
var node *Node
|
||||
for i := range s.Nodes {
|
||||
if s.Nodes[i].Name == nodeName {
|
||||
node = &(s.Nodes[i])
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if node == nil {
|
||||
// a node with the given name was not found
|
||||
return nil
|
||||
}
|
||||
|
||||
// now resolve all properties that haven't already been set on the node
|
||||
s.resolveNodeSelection(node)
|
||||
s.resolveNodeConfig(node)
|
||||
|
||||
return node
|
||||
}
|
||||
|
||||
func (s *StorageSpec) resolveNodeSelection(node *Node) {
|
||||
resolveString(&(node.Selection.DeviceFilter), s.Selection.DeviceFilter, "")
|
||||
resolveString(&(node.Selection.MetadataDevice), s.Selection.MetadataDevice, "")
|
||||
|
||||
if node.Selection.UseAllDevices == nil {
|
||||
if s.Selection.UseAllDevices != nil {
|
||||
// the node does not have a value specified for use all devices, but the cluster does. Use the cluster's.
|
||||
node.Selection.UseAllDevices = s.Selection.UseAllDevices
|
||||
} else {
|
||||
// neither node nor cluster have a value set for use all devices, use the default value.
|
||||
node.Selection.UseAllDevices = newBool(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StorageSpec) resolveNodeConfig(node *Node) {
|
||||
resolveString(&(node.Config.StoreConfig.StoreType), s.Config.StoreConfig.StoreType, cephosd.DefaultStore)
|
||||
resolveInt(&(node.Config.StoreConfig.DatabaseSizeMB), s.Config.StoreConfig.DatabaseSizeMB, 0)
|
||||
resolveInt(&(node.Config.StoreConfig.WalSizeMB), s.Config.StoreConfig.WalSizeMB, 0)
|
||||
resolveInt(&(node.Config.StoreConfig.JournalSizeMB), s.Config.StoreConfig.JournalSizeMB, 0)
|
||||
resolveString(&(node.Config.Location), s.Config.Location, "")
|
||||
}
|
||||
|
||||
func (s *Selection) getUseAllDevices() bool {
|
||||
return s.UseAllDevices != nil && *(s.UseAllDevices)
|
||||
}
|
||||
|
||||
func resolveString(setting *string, parent, defaultVal string) {
|
||||
if *setting == "" {
|
||||
if parent != "" {
|
||||
*setting = parent
|
||||
} else {
|
||||
*setting = defaultVal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func resolveInt(setting *int, parent, defaultVal int) {
|
||||
if *setting == 0 {
|
||||
if parent != 0 {
|
||||
*setting = parent
|
||||
} else {
|
||||
*setting = defaultVal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newBool(val bool) *bool {
|
||||
return &val
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package osd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
cephosd "github.com/rook/rook/pkg/cephmgr/osd"
|
||||
)
|
||||
|
||||
func TestResolveNodeNotExist(t *testing.T) {
|
||||
|
||||
// a non existing node should return nil
|
||||
storageSpec := StorageSpec{}
|
||||
node := storageSpec.resolveNode("fake node")
|
||||
assert.Nil(t, node)
|
||||
|
||||
// a node with no properties defined should inherit them from the cluster storage spec
|
||||
storageSpec = StorageSpec{
|
||||
Selection: Selection{
|
||||
DeviceFilter: "^sd.",
|
||||
MetadataDevice: "nvme01",
|
||||
},
|
||||
Config: Config{
|
||||
Location: "root=default,row=a,rack=a2,chassis=a2a,host=a2a1",
|
||||
StoreConfig: cephosd.StoreConfig{
|
||||
StoreType: cephosd.Bluestore,
|
||||
DatabaseSizeMB: 1024,
|
||||
WalSizeMB: 128,
|
||||
JournalSizeMB: 2048,
|
||||
},
|
||||
},
|
||||
Nodes: []Node{
|
||||
{Name: "node1"}, // node gets nothing but its name set
|
||||
},
|
||||
}
|
||||
node = storageSpec.resolveNode("node1")
|
||||
assert.NotNil(t, node)
|
||||
assert.Equal(t, "^sd.", node.Selection.DeviceFilter)
|
||||
assert.Equal(t, "nvme01", node.Selection.MetadataDevice)
|
||||
assert.False(t, node.Selection.getUseAllDevices())
|
||||
assert.Equal(t, "root=default,row=a,rack=a2,chassis=a2a,host=a2a1", node.Config.Location)
|
||||
assert.Equal(t, cephosd.Bluestore, node.Config.StoreConfig.StoreType)
|
||||
assert.Equal(t, 1024, node.Config.StoreConfig.DatabaseSizeMB)
|
||||
assert.Equal(t, 128, node.Config.StoreConfig.WalSizeMB)
|
||||
assert.Equal(t, 2048, node.Config.StoreConfig.JournalSizeMB)
|
||||
}
|
||||
|
||||
func TestResolveNodeInherentFromCluster(t *testing.T) {
|
||||
// a node with no properties defined should inherit them from the cluster storage spec
|
||||
storageSpec := StorageSpec{
|
||||
Selection: Selection{
|
||||
DeviceFilter: "^sd.",
|
||||
MetadataDevice: "nvme01",
|
||||
},
|
||||
Config: Config{
|
||||
Location: "root=default,row=a,rack=a2,chassis=a2a,host=a2a1",
|
||||
StoreConfig: cephosd.StoreConfig{
|
||||
StoreType: cephosd.Bluestore,
|
||||
DatabaseSizeMB: 1024,
|
||||
WalSizeMB: 128,
|
||||
JournalSizeMB: 2048,
|
||||
},
|
||||
},
|
||||
Nodes: []Node{
|
||||
{Name: "node1"}, // node gets nothing but its name set
|
||||
},
|
||||
}
|
||||
node := storageSpec.resolveNode("node1")
|
||||
assert.NotNil(t, node)
|
||||
assert.Equal(t, "^sd.", node.Selection.DeviceFilter)
|
||||
assert.Equal(t, "nvme01", node.Selection.MetadataDevice)
|
||||
assert.False(t, node.Selection.getUseAllDevices())
|
||||
assert.Equal(t, "root=default,row=a,rack=a2,chassis=a2a,host=a2a1", node.Config.Location)
|
||||
assert.Equal(t, cephosd.Bluestore, node.Config.StoreConfig.StoreType)
|
||||
assert.Equal(t, 1024, node.Config.StoreConfig.DatabaseSizeMB)
|
||||
assert.Equal(t, 128, node.Config.StoreConfig.WalSizeMB)
|
||||
assert.Equal(t, 2048, node.Config.StoreConfig.JournalSizeMB)
|
||||
}
|
||||
|
||||
func TestResolveNodeDefaultValues(t *testing.T) {
|
||||
|
||||
// a node with no properties and none defined in the cluster storage spec should get the default values
|
||||
storageSpec := StorageSpec{
|
||||
Nodes: []Node{
|
||||
{Name: "node1"}, // node gets nothing but its name set
|
||||
},
|
||||
}
|
||||
|
||||
node := storageSpec.resolveNode("node1")
|
||||
assert.NotNil(t, node)
|
||||
assert.Equal(t, "", node.Selection.DeviceFilter)
|
||||
assert.Equal(t, "", node.Selection.MetadataDevice)
|
||||
assert.False(t, node.Selection.getUseAllDevices())
|
||||
assert.Equal(t, "", node.Config.Location)
|
||||
assert.Equal(t, cephosd.Filestore, node.Config.StoreConfig.StoreType)
|
||||
assert.Equal(t, 0, node.Config.StoreConfig.DatabaseSizeMB)
|
||||
assert.Equal(t, 0, node.Config.StoreConfig.WalSizeMB)
|
||||
assert.Equal(t, 0, node.Config.StoreConfig.JournalSizeMB)
|
||||
}
|
||||
|
||||
func TestResolveNodeUseAllDevices(t *testing.T) {
|
||||
|
||||
storageSpec := StorageSpec{
|
||||
Selection: Selection{UseAllDevices: newBool(true)}, // UseAllDevices is set to true on the storage spec
|
||||
Nodes: []Node{
|
||||
{Name: "node1"}, // node gets nothing but its name set
|
||||
},
|
||||
}
|
||||
|
||||
node := storageSpec.resolveNode("node1")
|
||||
assert.NotNil(t, node)
|
||||
assert.True(t, node.Selection.getUseAllDevices())
|
||||
}
|
||||
|
||||
func TestUseAllDevices(t *testing.T) {
|
||||
storageSpec := StorageSpec{}
|
||||
assert.False(t, storageSpec.AnyUseAllDevices())
|
||||
|
||||
storageSpec = StorageSpec{
|
||||
Selection: Selection{
|
||||
UseAllDevices: newBool(true)}, // UseAllDevices is set to true on the storage spec
|
||||
}
|
||||
assert.True(t, storageSpec.AnyUseAllDevices())
|
||||
|
||||
storageSpec = StorageSpec{
|
||||
Selection: Selection{UseAllDevices: newBool(false)},
|
||||
Nodes: []Node{
|
||||
{
|
||||
Name: "node1",
|
||||
Selection: Selection{UseAllDevices: newBool(true)},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.True(t, storageSpec.AnyUseAllDevices())
|
||||
}
|
||||
|
||||
func TestClearUseAllDevices(t *testing.T) {
|
||||
// create a storage spec with use all devices set to true for the cluster and for all nodes
|
||||
storageSpec := StorageSpec{
|
||||
Selection: Selection{UseAllDevices: newBool(true)},
|
||||
Nodes: []Node{
|
||||
{
|
||||
Name: "node1",
|
||||
Selection: Selection{UseAllDevices: newBool(true)},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.True(t, storageSpec.AnyUseAllDevices())
|
||||
|
||||
// now clear the use all devices field, it should be cleared from the entire cluster and its nodes
|
||||
storageSpec.ClearUseAllDevices()
|
||||
assert.False(t, storageSpec.AnyUseAllDevices())
|
||||
}
|
||||
@@ -158,7 +158,7 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
|
||||
|
||||
func (c *Cluster) rgwContainer() v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --rgw-port=%d --rgw-host=%s",
|
||||
command := fmt.Sprintf("/usr/bin/rookd rgw --config-dir=%s --rgw-port=%d --rgw-host=%s",
|
||||
k8sutil.DataDir, cephrgw.RGWPort, cephrgw.DNSName)
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
|
||||
@@ -95,7 +95,7 @@ func TestPodSpecs(t *testing.T) {
|
||||
assert.Equal(t, 1, len(cont.VolumeMounts))
|
||||
assert.Equal(t, 5, len(cont.Env))
|
||||
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=/var/lib/rook --rgw-port=%d --rgw-host=%s",
|
||||
expectedCommand := fmt.Sprintf("/usr/bin/rookd rgw --config-dir=/var/lib/rook --rgw-port=%d --rgw-host=%s",
|
||||
cephrgw.RGWPort, cephrgw.DNSName)
|
||||
|
||||
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
|
||||
|
||||
@@ -18,6 +18,7 @@ package exec
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
@@ -30,6 +31,7 @@ type Executor interface {
|
||||
ExecuteCommandPipeline(actionName string, command string) (string, error)
|
||||
ExecuteCommandWithOutput(actionName string, command string, arg ...string) (string, error)
|
||||
ExecuteCommandWithCombinedOutput(actionName string, command string, arg ...string) (string, error)
|
||||
ExecuteStat(name string) (os.FileInfo, error)
|
||||
}
|
||||
|
||||
type CommandExecutor struct {
|
||||
@@ -93,6 +95,10 @@ func startCommand(command string, arg ...string) (*exec.Cmd, io.ReadCloser, io.R
|
||||
return cmd, stdout, stderr, err
|
||||
}
|
||||
|
||||
func (*CommandExecutor) ExecuteStat(name string) (os.FileInfo, error) {
|
||||
return os.Stat(name)
|
||||
}
|
||||
|
||||
func logOutput(name string, stdout, stderr io.ReadCloser) {
|
||||
if stdout == nil || stderr == nil {
|
||||
logger.Warningf("failed to collect stdout and stderr")
|
||||
|
||||
@@ -15,7 +15,10 @@ limitations under the License.
|
||||
*/
|
||||
package test
|
||||
|
||||
import "os/exec"
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
// ******************** MockExecutor ********************
|
||||
type MockExecutor struct {
|
||||
@@ -24,6 +27,7 @@ type MockExecutor struct {
|
||||
MockExecuteCommandPipeline func(actionName string, command string) (string, error)
|
||||
MockExecuteCommandWithOutput func(actionName string, command string, arg ...string) (string, error)
|
||||
MockExecuteCommandWithCombinedOutput func(actionName string, command string, arg ...string) (string, error)
|
||||
MockExecuteStat func(name string) (os.FileInfo, error)
|
||||
}
|
||||
|
||||
func (e *MockExecutor) ExecuteCommand(actionName string, command string, arg ...string) error {
|
||||
@@ -66,3 +70,11 @@ func (e *MockExecutor) ExecuteCommandWithCombinedOutput(actionName string, comma
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (e *MockExecutor) ExecuteStat(name string) (os.FileInfo, error) {
|
||||
if e.MockExecuteStat != nil {
|
||||
return e.MockExecuteStat(name)
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func CreatePartitions(device string, args []string, executor exec.Executor) erro
|
||||
|
||||
func FormatDevice(devicePath string, executor exec.Executor) error {
|
||||
cmd := fmt.Sprintf("mkfs.ext4 %s", devicePath)
|
||||
if err := executor.ExecuteCommand(cmd, "sudo", "mkfs.ext4", devicePath); err != nil {
|
||||
if err := executor.ExecuteCommand(cmd, "mkfs.ext4", devicePath); err != nil {
|
||||
return fmt.Errorf("command %s failed: %+v", cmd, err)
|
||||
}
|
||||
|
||||
@@ -197,7 +197,7 @@ func MountDevice(devicePath, mountPath string, executor exec.Executor) error {
|
||||
|
||||
// comma-separated list of mount options passed directly to mount command
|
||||
func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, executor exec.Executor) error {
|
||||
args := []string{"mount"}
|
||||
args := []string{}
|
||||
|
||||
if fstype != "" {
|
||||
args = append(args, "-t", fstype)
|
||||
@@ -212,7 +212,7 @@ func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, execu
|
||||
|
||||
os.MkdirAll(mountPath, 0755)
|
||||
cmd := fmt.Sprintf("mount %s", devicePath)
|
||||
if err := executor.ExecuteCommand(cmd, "sudo", args...); err != nil {
|
||||
if err := executor.ExecuteCommand(cmd, "mount", args...); err != nil {
|
||||
return fmt.Errorf("command %s failed: %+v", cmd, err)
|
||||
}
|
||||
|
||||
@@ -221,7 +221,7 @@ func MountDeviceWithOptions(devicePath, mountPath, fstype, options string, execu
|
||||
|
||||
func UnmountDevice(devicePath string, executor exec.Executor) error {
|
||||
cmd := fmt.Sprintf("umount %s", devicePath)
|
||||
if err := executor.ExecuteCommand(cmd, "sudo", "umount", devicePath); err != nil {
|
||||
if err := executor.ExecuteCommand(cmd, "umount", devicePath); err != nil {
|
||||
cmdErr, ok := err.(*exec.CommandError)
|
||||
if ok && cmdErr.ExitStatus() == 32 {
|
||||
logger.Infof("ignoring exit status 32 from unmount of device %s, err:%+v", devicePath, cmdErr)
|
||||
|
||||
@@ -94,13 +94,13 @@ func TestMountDeviceWithOptions(t *testing.T) {
|
||||
MockExecuteCommand: func(actionName string, command string, arg ...string) error {
|
||||
switch testCount {
|
||||
case 0:
|
||||
assert.Equal(t, []string{"mount", "/dev/abc1", "/tmp/mount1"}, arg)
|
||||
assert.Equal(t, []string{"/dev/abc1", "/tmp/mount1"}, arg)
|
||||
case 1:
|
||||
assert.Equal(t, []string{"mount", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg)
|
||||
assert.Equal(t, []string{"-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg)
|
||||
case 2:
|
||||
assert.Equal(t, []string{"mount", "-t", "myfstype", "/dev/abc1", "/tmp/mount1"}, arg)
|
||||
assert.Equal(t, []string{"-t", "myfstype", "/dev/abc1", "/tmp/mount1"}, arg)
|
||||
case 3:
|
||||
assert.Equal(t, []string{"mount", "-t", "myfstype", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg)
|
||||
assert.Equal(t, []string{"-t", "myfstype", "-o", "foo=bar,baz=biz", "/dev/abc1", "/tmp/mount1"}, arg)
|
||||
}
|
||||
|
||||
testCount++
|
||||
|
||||
Reference in new issue
Block a user