osd: bluestore is the default for devices and filestore for dirs

Signed-off-by: travisn <tnielsen@redhat.com>

Documentation/cluster-crd.md

@@ -72,7 +72,7 @@ Below are the settings available, both at the cluster and individual node level,
 Below are the settings available, both at the cluster and individual node level, that affect how the selected storage resources will be configured.
 - `location`: Location information about the cluster to help with data placement, such as region or data center.  This is directly fed into the underlying Ceph CRUSH map.  More information on CRUSH maps can be found in the [ceph docs](http://docs.ceph.com/docs/master/rados/operations/crush-map/).
 - `storeConfig`: Configuration information about the store format for each OSD.
-  - `storeType`: `filestore` or `bluestore` (default: `bluestore`), The underlying storage format to use for each OSD.
+  - `storeType`: `filestore` or `bluestore`, the underlying storage format to use for each OSD. The default is set dynamically to `bluestore` for devices, while `filestore` is the default for directories. Set this store type explicitly to override the default. Warning: Bluestore is **not** recommended for directories in production. Bluestore does not purge data from the directory and over time will grow without the ability to compact or shrink.
   - `databaseSizeMB`:  The size in MB of a bluestore database.
   - `walSizeMB`:  The size in MB of a bluestore write ahead log (WAL).
   - `journalSizeMB`:  The size in MB of a filestore journal.
@@ -136,7 +136,6 @@ spec:
     metadataDevice:
     location:
     storeConfig:
-      storeType: bluestore
       databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger)
       journalSizeMB: 1024  # this value can be removed for environments with normal sized disks (20 GB or larger)
 ```
@@ -167,7 +166,6 @@ spec:
     metadataDevice:
     location:
     storeConfig:
-      storeType: bluestore
       databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger)
       journalSizeMB: 1024  # this value can be removed for environments with normal sized disks (20 GB or larger)
     nodes:
@@ -179,7 +177,7 @@ spec:
       - name: "sdb"
       - name: "sdc"
       storeConfig:         # configuration can be specified at the node level which overrides the cluster level config
-        storeType: filestore
+        storeType: bluestore
     - name: "172.17.4.301"
       deviceFilter: "^sd."
 ```
@@ -207,7 +205,6 @@ spec:
     useAllNodes: false
     useAllDevices: false
     storeConfig:
-      storeType: bluestore
       databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger)
       journalSizeMB: 1024  # this value can be removed for environments with normal sized disks (20 GB or larger)
     directories:

cluster/examples/kubernetes/rook-cluster.yaml

@@ -69,7 +69,9 @@ spec:
     metadataDevice:
     location:
     storeConfig:
-      storeType: bluestore
+      # The default and recommended storeType is dynamically set to bluestore for devices and filestore for directories.
+      # Set the storeType explicitly only if it is required not to use the default.
+      # storeType: bluestore 
       databaseSizeMB: 1024 # this value can be removed for environments with normal sized disks (100 GB or larger)
       journalSizeMB: 1024  # this value can be removed for environments with normal sized disks (20 GB or larger)
 # Cluster level list of directories to use for storage. These values will be set for all nodes that have no `directories` set.

cmd/rook/osd.go

@@ -56,7 +56,7 @@ func addOSDFlags(command *cobra.Command) {
 	command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osdcfg.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)")
 	command.Flags().IntVar(&cfg.storeConfig.DatabaseSizeMB, "osd-database-size", osdcfg.DBDefaultSizeMB, "default size (MB) for OSD database (bluestore)")
 	command.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osdcfg.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)")
-	command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", osdcfg.DefaultStore, "type of backing OSD store to use (bluestore or filestore)")
+	command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)")
 }
 
 func init() {

pkg/daemon/ceph/osd/agent.go

@@ -654,10 +654,12 @@ func isBluestore(config *osdConfig) bool {
 }
 
 func isBluestoreDevice(cfg *osdConfig) bool {
-	return !cfg.dir && cfg.partitionScheme != nil && cfg.partitionScheme.StoreType == config.Bluestore
+	// A device will use bluestore unless explicitly requested to be filestore (the default is blank)
+	return !cfg.dir && cfg.partitionScheme != nil && cfg.partitionScheme.StoreType != config.Filestore
 }
 
 func isBluestoreDir(cfg *osdConfig) bool {
+	// A dir will use filestore unless explicitly requested to be bluestore
 	return cfg.dir && cfg.storeConfig.StoreType == config.Bluestore
 }
 
@@ -666,9 +668,11 @@ func isFilestore(cfg *osdConfig) bool {
 }
 
 func isFilestoreDevice(cfg *osdConfig) bool {
+	// A device will use bluestore unless explicitly requested to be filestore (the default is blank)
 	return !cfg.dir && cfg.partitionScheme != nil && cfg.partitionScheme.StoreType == config.Filestore
 }
 
 func isFilestoreDir(cfg *osdConfig) bool {
-	return cfg.dir && cfg.storeConfig.StoreType == config.Filestore
+	// A dir will use filestore unless explicitly requested to be bluestore (the default is blank)
+	return cfg.dir && cfg.storeConfig.StoreType != config.Bluestore
 }

pkg/daemon/ceph/osd/agent_test.go

@@ -39,6 +39,44 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+func TestStoreTypeDefaults(t *testing.T) {
+	// A filestore dir
+	cfg := &osdConfig{dir: true, storeConfig: rookalpha.StoreConfig{StoreType: ""}}
+	assert.True(t, isFilestore(cfg))
+	assert.False(t, isFilestoreDevice(cfg))
+	assert.True(t, isFilestoreDir(cfg))
+	assert.False(t, isBluestore(cfg))
+	assert.False(t, isBluestoreDevice(cfg))
+	assert.False(t, isBluestoreDir(cfg))
+
+	// A bluestore dir
+	cfg = &osdConfig{dir: true, storeConfig: rookalpha.StoreConfig{StoreType: "bluestore"}}
+	assert.False(t, isFilestore(cfg))
+	assert.False(t, isFilestoreDevice(cfg))
+	assert.False(t, isFilestoreDir(cfg))
+	assert.True(t, isBluestore(cfg))
+	assert.False(t, isBluestoreDevice(cfg))
+	assert.True(t, isBluestoreDir(cfg))
+
+	// a bluestore device
+	cfg = &osdConfig{dir: false, partitionScheme: &config.PerfSchemeEntry{StoreType: ""}}
+	assert.False(t, isFilestore(cfg))
+	assert.False(t, isFilestoreDevice(cfg))
+	assert.False(t, isFilestoreDir(cfg))
+	assert.True(t, isBluestore(cfg))
+	assert.True(t, isBluestoreDevice(cfg))
+	assert.False(t, isBluestoreDir(cfg))
+
+	// A filestore device
+	cfg = &osdConfig{dir: false, partitionScheme: &config.PerfSchemeEntry{StoreType: "filestore"}}
+	assert.True(t, isFilestore(cfg))
+	assert.True(t, isFilestoreDevice(cfg))
+	assert.False(t, isFilestoreDir(cfg))
+	assert.False(t, isBluestore(cfg))
+	assert.False(t, isBluestoreDevice(cfg))
+	assert.False(t, isBluestoreDir(cfg))
+}
+
 func TestOSDAgentWithDevicesFilestore(t *testing.T) {
 	testOSDAgentWithDevicesHelper(t, rookalpha.StoreConfig{StoreType: config.Filestore})
 }

pkg/operator/cluster/ceph/osd/config/scheme.go

@@ -28,7 +28,6 @@ import (
 const (
 	Filestore             = "filestore"
 	Bluestore             = "bluestore"
-	DefaultStore          = Bluestore
 	UseRemainingSpace     = -1
 	schemeKeyName         = "partition-scheme"
 	WalDefaultSizeMB      = 576