From 3840dd1c6613fbbdc809b860fbebd2a08025f0ff Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 20 Mar 2018 15:39:42 -0700 Subject: [PATCH] osd: bluestore is the default for devices and filestore for dirs Signed-off-by: travisn --- Documentation/cluster-crd.md | 7 +--- cluster/examples/kubernetes/rook-cluster.yaml | 4 +- cmd/rook/osd.go | 2 +- pkg/daemon/ceph/osd/agent.go | 8 +++- pkg/daemon/ceph/osd/agent_test.go | 38 +++++++++++++++++++ .../cluster/ceph/osd/config/scheme.go | 1 - 6 files changed, 50 insertions(+), 10 deletions(-) diff --git a/Documentation/cluster-crd.md b/Documentation/cluster-crd.md index 4daf1d51e..5728f9c05 100644 --- a/Documentation/cluster-crd.md +++ b/Documentation/cluster-crd.md @@ -72,7 +72,7 @@ Below are the settings available, both at the cluster and individual node level, Below are the settings available, both at the cluster and individual node level, that affect how the selected storage resources will be configured. - `location`: Location information about the cluster to help with data placement, such as region or data center. This is directly fed into the underlying Ceph CRUSH map. More information on CRUSH maps can be found in the [ceph docs](http://docs.ceph.com/docs/master/rados/operations/crush-map/). - `storeConfig`: Configuration information about the store format for each OSD. - - `storeType`: `filestore` or `bluestore` (default: `bluestore`), The underlying storage format to use for each OSD. + - `storeType`: `filestore` or `bluestore`, the underlying storage format to use for each OSD. The default is set dynamically to `bluestore` for devices, while `filestore` is the default for directories. Set this store type explicitly to override the default. Warning: Bluestore is **not** recommended for directories in production. Bluestore does not purge data from the directory and over time will grow without the ability to compact or shrink. - `databaseSizeMB`: The size in MB of a bluestore database. - `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). - `journalSizeMB`: The size in MB of a filestore journal. @@ -136,7 +136,6 @@ spec: metadataDevice: location: storeConfig: - storeType: bluestore databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger) journalSizeMB: 1024 # this value can be removed for environments with normal sized disks (20 GB or larger) ``` @@ -167,7 +166,6 @@ spec: metadataDevice: location: storeConfig: - storeType: bluestore databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger) journalSizeMB: 1024 # this value can be removed for environments with normal sized disks (20 GB or larger) nodes: @@ -179,7 +177,7 @@ spec: - name: "sdb" - name: "sdc" storeConfig: # configuration can be specified at the node level which overrides the cluster level config - storeType: filestore + storeType: bluestore - name: "172.17.4.301" deviceFilter: "^sd." ``` @@ -207,7 +205,6 @@ spec: useAllNodes: false useAllDevices: false storeConfig: - storeType: bluestore databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger) journalSizeMB: 1024 # this value can be removed for environments with normal sized disks (20 GB or larger) directories: diff --git a/cluster/examples/kubernetes/rook-cluster.yaml b/cluster/examples/kubernetes/rook-cluster.yaml index bc7e8510a..5c5e2a783 100644 --- a/cluster/examples/kubernetes/rook-cluster.yaml +++ b/cluster/examples/kubernetes/rook-cluster.yaml @@ -69,7 +69,9 @@ spec: metadataDevice: location: storeConfig: - storeType: bluestore + # The default and recommended storeType is dynamically set to bluestore for devices and filestore for directories. + # Set the storeType explicitly only if it is required not to use the default. + # storeType: bluestore databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger) journalSizeMB: 1024 # this value can be removed for environments with normal sized disks (20 GB or larger) # Cluster level list of directories to use for storage. These values will be set for all nodes that have no `directories` set. diff --git a/cmd/rook/osd.go b/cmd/rook/osd.go index 92f801a83..fb398e5a6 100644 --- a/cmd/rook/osd.go +++ b/cmd/rook/osd.go @@ -56,7 +56,7 @@ func addOSDFlags(command *cobra.Command) { command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osdcfg.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)") command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", osdcfg.DBDefaultSizeMB, "default size (MB) for OSD database (bluestore)") command.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osdcfg.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)") - command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", osdcfg.DefaultStore, "type of backing OSD store to use (bluestore or filestore)") + command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)") } func init() { diff --git a/pkg/daemon/ceph/osd/agent.go b/pkg/daemon/ceph/osd/agent.go index c08e9c6f6..639bb28f8 100644 --- a/pkg/daemon/ceph/osd/agent.go +++ b/pkg/daemon/ceph/osd/agent.go @@ -654,10 +654,12 @@ func isBluestore(config *osdConfig) bool { } func isBluestoreDevice(cfg *osdConfig) bool { - return !cfg.dir && cfg.partitionScheme != nil && cfg.partitionScheme.StoreType == config.Bluestore + // A device will use bluestore unless explicitly requested to be filestore (the default is blank) + return !cfg.dir && cfg.partitionScheme != nil && cfg.partitionScheme.StoreType != config.Filestore } func isBluestoreDir(cfg *osdConfig) bool { + // A dir will use filestore unless explicitly requested to be bluestore return cfg.dir && cfg.storeConfig.StoreType == config.Bluestore } @@ -666,9 +668,11 @@ func isFilestore(cfg *osdConfig) bool { } func isFilestoreDevice(cfg *osdConfig) bool { + // A device will use bluestore unless explicitly requested to be filestore (the default is blank) return !cfg.dir && cfg.partitionScheme != nil && cfg.partitionScheme.StoreType == config.Filestore } func isFilestoreDir(cfg *osdConfig) bool { - return cfg.dir && cfg.storeConfig.StoreType == config.Filestore + // A dir will use filestore unless explicitly requested to be bluestore (the default is blank) + return cfg.dir && cfg.storeConfig.StoreType != config.Bluestore } diff --git a/pkg/daemon/ceph/osd/agent_test.go b/pkg/daemon/ceph/osd/agent_test.go index 863e5e11c..de8282e0b 100644 --- a/pkg/daemon/ceph/osd/agent_test.go +++ b/pkg/daemon/ceph/osd/agent_test.go @@ -39,6 +39,44 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func TestStoreTypeDefaults(t *testing.T) { + // A filestore dir + cfg := &osdConfig{dir: true, storeConfig: rookalpha.StoreConfig{StoreType: ""}} + assert.True(t, isFilestore(cfg)) + assert.False(t, isFilestoreDevice(cfg)) + assert.True(t, isFilestoreDir(cfg)) + assert.False(t, isBluestore(cfg)) + assert.False(t, isBluestoreDevice(cfg)) + assert.False(t, isBluestoreDir(cfg)) + + // A bluestore dir + cfg = &osdConfig{dir: true, storeConfig: rookalpha.StoreConfig{StoreType: "bluestore"}} + assert.False(t, isFilestore(cfg)) + assert.False(t, isFilestoreDevice(cfg)) + assert.False(t, isFilestoreDir(cfg)) + assert.True(t, isBluestore(cfg)) + assert.False(t, isBluestoreDevice(cfg)) + assert.True(t, isBluestoreDir(cfg)) + + // a bluestore device + cfg = &osdConfig{dir: false, partitionScheme: &config.PerfSchemeEntry{StoreType: ""}} + assert.False(t, isFilestore(cfg)) + assert.False(t, isFilestoreDevice(cfg)) + assert.False(t, isFilestoreDir(cfg)) + assert.True(t, isBluestore(cfg)) + assert.True(t, isBluestoreDevice(cfg)) + assert.False(t, isBluestoreDir(cfg)) + + // A filestore device + cfg = &osdConfig{dir: false, partitionScheme: &config.PerfSchemeEntry{StoreType: "filestore"}} + assert.True(t, isFilestore(cfg)) + assert.True(t, isFilestoreDevice(cfg)) + assert.False(t, isFilestoreDir(cfg)) + assert.False(t, isBluestore(cfg)) + assert.False(t, isBluestoreDevice(cfg)) + assert.False(t, isBluestoreDir(cfg)) +} + func TestOSDAgentWithDevicesFilestore(t *testing.T) { testOSDAgentWithDevicesHelper(t, rookalpha.StoreConfig{StoreType: config.Filestore}) } diff --git a/pkg/operator/cluster/ceph/osd/config/scheme.go b/pkg/operator/cluster/ceph/osd/config/scheme.go index 26b83a90b..0bd41117d 100644 --- a/pkg/operator/cluster/ceph/osd/config/scheme.go +++ b/pkg/operator/cluster/ceph/osd/config/scheme.go @@ -28,7 +28,6 @@ import ( const ( Filestore = "filestore" Bluestore = "bluestore" - DefaultStore = Bluestore UseRemainingSpace = -1 schemeKeyName = "partition-scheme" WalDefaultSizeMB = 576