Merge pull request #12589 from rook/mergify/bp/release-1.12/pr-12466

osd: configure backend store (backport #12466)

Documentation/CRDs/specification.md

@@ -8181,6 +8181,36 @@ string
 </tr>
 </tbody>
 </table>
+<h3 id="ceph.rook.io/v1.OSDStore">OSDStore
+</h3>
+<p>
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.StorageScopeSpec">StorageScopeSpec</a>)
+</p>
+<div>
+<p>OSDStore is the backend storage type used for creating the OSDs</p>
+</div>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>type</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="ceph.rook.io/v1.ObjectEndpoints">ObjectEndpoints
 </h3>
 <p>
@@ -11135,8 +11165,40 @@ Selection
 <em>(Optional)</em>
 </td>
 </tr>
+<tr>
+<td>
+<code>store</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.OSDStore">
+OSDStore
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+</td>
+</tr>
 </tbody>
 </table>
+<h3 id="ceph.rook.io/v1.StoreType">StoreType
+(<code>string</code> alias)</h3>
+<div>
+</div>
+<table>
+<thead>
+<tr>
+<th>Value</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody><tr><td><p>&#34;bluestore&#34;</p></td>
+<td><p>StoreTypeBlueStore is the bluestore backend storage for OSDs</p>
+</td>
+</tr><tr><td><p>&#34;bluestore-rdr&#34;</p></td>
+<td><p>StoreTypeBlueStoreRDR is the bluestore-rdr backed storage for OSDs</p>
+</td>
+</tr></tbody>
+</table>
 <h3 id="ceph.rook.io/v1.StretchClusterSpec">StretchClusterSpec
 </h3>
 <p>

cmd/rook/ceph/osd.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/cmd/rook/rook"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	osddaemon "github.com/rook/rook/pkg/daemon/ceph/osd"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd"
@@ -131,6 +132,7 @@ func addOSDConfigFlags(command *cobra.Command) {
 	command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt")
 	command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node")
 	command.Flags().StringVar(&cfg.storeConfig.InitialWeight, "osd-crush-initial-weight", "", "The initial weight of OSD in TiB units")
+	command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store-type", string(cephv1.StoreTypeBlueStore), "the osd store type such as bluestore")
 }
 
 func init() {

deploy/charts/rook-ceph/templates/resources.yaml

@@ -4360,6 +4360,16 @@ spec:
                         type: object
                       nullable: true
                       type: array
+                    store:
+                      description: OSDStore is the backend storage type used for creating the OSDs
+                      properties:
+                        type:
+                          description: Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used
+                          enum:
+                            - bluestore
+                            - bluestore-rdr
+                          type: string
+                      type: object
                     useAllDevices:
                       description: Whether to consume all the storage devices found on a machine
                       type: boolean

deploy/examples/crds.yaml

@@ -4358,6 +4358,16 @@ spec:
                         type: object
                       nullable: true
                       type: array
+                    store:
+                      description: OSDStore is the backend storage type used for creating the OSDs
+                      properties:
+                        type:
+                          description: Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used
+                          enum:
+                            - bluestore
+                            - bluestore-rdr
+                          type: string
+                      type: object
                     useAllDevices:
                       description: Whether to consume all the storage devices found on a machine
                       type: boolean

pkg/apis/ceph.rook.io/v1/storage.go

@@ -15,6 +15,18 @@ limitations under the License.
 */
 package v1
 
+import "fmt"
+
+type StoreType string
+
+const (
+	// StoreTypeBlueStore is the bluestore backend storage for OSDs
+	StoreTypeBlueStore StoreType = "bluestore"
+
+	// StoreTypeBlueStoreRDR is the bluestore-rdr backed storage for OSDs
+	StoreTypeBlueStoreRDR StoreType = "bluestore-rdr"
+)
+
 // AnyUseAllDevices gets whether to use all devices
 func (s *StorageScopeSpec) AnyUseAllDevices() bool {
 	if s.Selection.GetUseAllDevices() {
@@ -168,3 +180,19 @@ func (s *StorageScopeSpec) IsOnPVCEncrypted() bool {
 
 	return false
 }
+
+// GetOSDStore returns osd backend store type provided in the cluster spec
+func (s *StorageScopeSpec) GetOSDStore() string {
+	if s.Store.Type == "" {
+		return string(StoreTypeBlueStore)
+	}
+	return s.Store.Type
+}
+
+// GetOSDStoreFlag returns osd backend store type prefixed with "--"
+func (s *StorageScopeSpec) GetOSDStoreFlag() string {
+	if s.Store.Type == "" {
+		return fmt.Sprintf("--%s", StoreTypeBlueStore)
+	}
+	return fmt.Sprintf("--%s", s.Store.Type)
+}

pkg/apis/ceph.rook.io/v1/types.go

@@ -2587,6 +2587,16 @@ type StorageScopeSpec struct {
 	// +nullable
 	// +optional
 	StorageClassDeviceSets []StorageClassDeviceSet `json:"storageClassDeviceSets,omitempty"`
+	// +optional
+	Store OSDStore `json:"store,omitempty"`
+}
+
+// OSDStore is the backend storage type used for creating the OSDs
+type OSDStore struct {
+	// Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used
+	// +optional
+	// +kubebuilder:validation:Enum=bluestore;bluestore-rdr;
+	Type string `json:"type,omitempty"`
 }
 
 // Node is a storage nodes

pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go

@@ -3197,6 +3197,22 @@ func (in *NotificationKeyFilterRule) DeepCopy() *NotificationKeyFilterRule {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OSDStore) DeepCopyInto(out *OSDStore) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OSDStore.
+func (in *OSDStore) DeepCopy() *OSDStore {
+	if in == nil {
+		return nil
+	}
+	out := new(OSDStore)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ObjectEndpoints) DeepCopyInto(out *ObjectEndpoints) {
 	*out = *in
@@ -4217,6 +4233,7 @@ func (in *StorageScopeSpec) DeepCopyInto(out *StorageScopeSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	out.Store = in.Store
 	return
 }
 

pkg/daemon/ceph/osd/volume.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd"
@@ -220,6 +221,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 	// we need to return the block if raw mode is used and the lv if lvm mode
 	baseCommand := "stdbuf"
 	var baseArgs []string
+	storeFlag := a.storeConfig.GetStoreFlag()
 
 	// Create a specific log directory so that each prepare command will have its own log
 	// Only do this if nothing is present so that we don't override existing logs
@@ -227,10 +229,9 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 	err := os.MkdirAll(cvLogDir, 0750)
 	if err != nil {
 		logger.Errorf("failed to create ceph-volume log directory %q, continue with default %q. %v", cvLogDir, cephLogDir, err)
-		baseArgs = []string{"-oL", cephVolumeCmd, "raw", "prepare", "--bluestore"}
+		baseArgs = []string{"-oL", cephVolumeCmd, "raw", "prepare", storeFlag}
 	} else {
-		// Always force Bluestore!
-		baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir, "raw", "prepare", "--bluestore"}
+		baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir, "raw", "prepare", storeFlag}
 	}
 
 	var metadataArg, walArg []string
@@ -511,7 +512,9 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 func (a *OsdAgent) initializeDevicesRawMode(context *clusterd.Context, devices *DeviceOsdMapping) error {
 	baseCommand := "stdbuf"
 	cephVolumeMode := "raw"
-	baseArgs := []string{"-oL", cephVolumeCmd, cephVolumeMode, "prepare", "--bluestore"}
+	storeFlag := a.storeConfig.GetStoreFlag()
+
+	baseArgs := []string{"-oL", cephVolumeCmd, cephVolumeMode, "prepare", storeFlag}
 
 	for name, device := range devices.Entries {
 		deviceArg := path.Join("/dev", name)
@@ -550,8 +553,7 @@ func (a *OsdAgent) initializeDevicesRawMode(context *clusterd.Context, devices *
 }
 
 func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *DeviceOsdMapping) error {
-	storeFlag := "--bluestore"
-
+	storeFlag := a.storeConfig.GetStoreFlag()
 	logPath := "/tmp/ceph-log"
 	if err := os.MkdirAll(logPath, 0700); err != nil {
 		return errors.Wrapf(err, "failed to create dir %q", logPath)
@@ -628,7 +630,7 @@ func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *
 					metadataDevices[md]["devices"] = deviceArg
 				}
 				deviceDBSizeMB := getDatabaseSize(a.storeConfig.DatabaseSizeMB, device.Config.DatabaseSizeMB)
-				if storeFlag == "--bluestore" && deviceDBSizeMB > 0 {
+				if a.storeConfig.IsValidStoreType() && deviceDBSizeMB > 0 {
 					if deviceDBSizeMB < cephVolumeMinDBSize {
 						// ceph-volume will convert this value to ?G. It needs to be > 1G to invoke lvcreate.
 						logger.Infof("skipping databaseSizeMB setting (%d). For it should be larger than %dMB.", deviceDBSizeMB, cephVolumeMinDBSize)
@@ -858,6 +860,11 @@ func GetCephVolumeLVMOSDs(context *clusterd.Context, clusterInfo *client.Cluster
 			lvPath = lv
 		}
 
+		osdStore := os.Getenv(oposd.OSDStoreTypeVarName)
+		if osdStore == "" {
+			osdStore = string(cephv1.StoreTypeBlueStore)
+		}
+
 		osd := oposd.OSDInfo{
 			ID:            id,
 			Cluster:       "ceph",
@@ -866,7 +873,7 @@ func GetCephVolumeLVMOSDs(context *clusterd.Context, clusterInfo *client.Cluster
 			SkipLVRelease: skipLVRelease,
 			LVBackedPV:    lvBackedPV,
 			CVMode:        cvMode,
-			Store:         "bluestore",
+			Store:         osdStore,
 			DeviceClass:   osdDeviceClass,
 		}
 		osds = append(osds, osd)
@@ -1046,6 +1053,11 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
 			blockPath = block
 		}
 
+		osdStore := os.Getenv(oposd.OSDStoreTypeVarName)
+		if osdStore == "" {
+			osdStore = string(cephv1.StoreTypeBlueStore)
+		}
+
 		osd := oposd.OSDInfo{
 			ID:      osdID,
 			Cluster: "ceph",
@@ -1061,7 +1073,7 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
 			SkipLVRelease: true,
 			LVBackedPV:    lvBackedPV,
 			CVMode:        cvMode,
-			Store:         "bluestore",
+			Store:         osdStore,
 			Encrypted:     strings.Contains(blockPath, "-dmcrypt"),
 		}
 

pkg/daemon/ceph/osd/volume_test.go

@@ -328,6 +328,8 @@ func TestConfigureCVDevices(t *testing.T) {
 		cephConfigDir = originalCephConfigDir
 	}()
 
+	os.Setenv(oposd.OSDStoreTypeVarName, "bluestore")
+
 	nodeName := "set1-data-0-rpf2k"
 	mountedDev := "/mnt/" + nodeName
 	mapperDev := "/dev/mapper/rook--vg-rook--lv1"
@@ -385,7 +387,7 @@ func TestConfigureCVDevices(t *testing.T) {
 			Context:     context.TODO(),
 		}
 		context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir}
-		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{DeviceClass: "myds"}}
+		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{DeviceClass: "myds", StoreType: "bluestore"}}
 		devices := createPVCAvailableDevices()
 		deviceOSDs, err := agent.configureCVDevices(context, devices)
 		assert.Nil(t, err)
@@ -438,7 +440,7 @@ func TestConfigureCVDevices(t *testing.T) {
 			CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0}, // It supports raw mode OSD
 			FSID:        clusterFSID,
 		}
-		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true}
+		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 		devices := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}}
 
 		deviceOSDs, err := agent.configureCVDevices(context, devices)
@@ -509,7 +511,7 @@ func TestConfigureCVDevices(t *testing.T) {
 			CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0}, // It supports raw mode OSD
 			FSID:        clusterFSID,
 		}
-		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true}
+		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 		devices := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}}
 
 		deviceOSDs, err := agent.configureCVDevices(context, devices)
@@ -562,7 +564,7 @@ func TestConfigureCVDevices(t *testing.T) {
 			CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 1}, // It supports raw mode OSD
 			FSID:        clusterFSID,
 		}
-		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, storeConfig: config.StoreConfig{DeviceClass: "myclass"}}
+		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, storeConfig: config.StoreConfig{DeviceClass: "myclass", StoreType: "bluestore"}}
 		devices := &DeviceOsdMapping{
 			Entries: map[string]*DeviceOsdIDEntry{
 				"vdb1": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/dev/vdb1"}, DeviceInfo: &sys.LocalDisk{Type: sys.PartType}},
@@ -632,6 +634,7 @@ func testBaseArgs(args []string) error {
 }
 
 func TestInitializeBlock(t *testing.T) {
+	os.Setenv(oposd.OSDStoreTypeVarName, "bluestore")
 	// Common vars for all the tests
 	devices := &DeviceOsdMapping{
 		Entries: map[string]*DeviceOsdIDEntry{
@@ -663,7 +666,7 @@ func TestInitializeBlock(t *testing.T) {
 
 			return errors.Errorf("unknown command %s %s", command, args)
 		}
-		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1"}
+		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 		context := &clusterd.Context{Executor: executor}
 
 		err := a.initializeDevicesLVMMode(context, devices)
@@ -695,7 +698,7 @@ func TestInitializeBlock(t *testing.T) {
 
 			return errors.Errorf("unknown command %s %s", command, args)
 		}
-		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{EncryptedDevice: true}}
+		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{EncryptedDevice: true, StoreType: "bluestore"}}
 		context := &clusterd.Context{Executor: executor}
 
 		err := a.initializeDevicesLVMMode(context, devices)
@@ -727,7 +730,7 @@ func TestInitializeBlock(t *testing.T) {
 
 			return errors.Errorf("unknown command %s %s", command, args)
 		}
-		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{OSDsPerDevice: 3}}
+		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{OSDsPerDevice: 3, StoreType: "bluestore"}}
 		context := &clusterd.Context{Executor: executor}
 
 		err := a.initializeDevicesLVMMode(context, devices)
@@ -759,7 +762,7 @@ func TestInitializeBlock(t *testing.T) {
 
 			return errors.Errorf("unknown command %s %s", command, args)
 		}
-		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{DeviceClass: "hybrid"}}
+		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{DeviceClass: "hybrid", StoreType: "bluestore"}}
 		context := &clusterd.Context{Executor: executor}
 		err := a.initializeDevicesLVMMode(context, devices)
 		assert.NoError(t, err, "failed crush device class test")
@@ -813,7 +816,7 @@ func TestInitializeBlock(t *testing.T) {
 
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
-		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1"}
+		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 		context := &clusterd.Context{
 			Executor: executor,
 			Devices: []*sys.LocalDisk{
@@ -878,7 +881,7 @@ func TestInitializeBlock(t *testing.T) {
 
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
-		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1"}
+		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 		context := &clusterd.Context{
 			Executor: executor,
 			Devices: []*sys.LocalDisk{
@@ -962,7 +965,8 @@ func TestInitializeBlock(t *testing.T) {
 			clusterInfo: &cephclient.ClusterInfo{
 				CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0},
 			},
-			nodeName: "node1",
+			nodeName:    "node1",
+			storeConfig: config.StoreConfig{StoreType: "bluestore"},
 		}
 		context := &clusterd.Context{Executor: executor,
 			Devices: []*sys.LocalDisk{
@@ -1043,7 +1047,8 @@ func TestInitializeBlock(t *testing.T) {
 			clusterInfo: &cephclient.ClusterInfo{
 				CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0},
 			},
-			nodeName: "node1",
+			nodeName:    "node1",
+			storeConfig: config.StoreConfig{StoreType: "bluestore"},
 		}
 		context := &clusterd.Context{Executor: executor,
 			Devices: []*sys.LocalDisk{
@@ -1086,7 +1091,8 @@ func TestInitializeBlock(t *testing.T) {
 			clusterInfo: &cephclient.ClusterInfo{
 				CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0},
 			},
-			nodeName: "node1",
+			nodeName:    "node1",
+			storeConfig: config.StoreConfig{StoreType: "bluestore"},
 		}
 		context := &clusterd.Context{Executor: executor,
 			Devices: []*sys.LocalDisk{
@@ -1126,7 +1132,7 @@ func TestInitializeBlockPVC(t *testing.T) {
 	clusterInfo := &cephclient.ClusterInfo{
 		CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0},
 	}
-	a := &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1"}
+	a := &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 	devices := &DeviceOsdMapping{
 		Entries: map[string]*DeviceOsdIDEntry{
 			"data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}},
@@ -1158,7 +1164,7 @@ func TestInitializeBlockPVC(t *testing.T) {
 	clusterInfo = &cephclient.ClusterInfo{
 		CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0},
 	}
-	a = &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1"}
+	a = &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 	devices = &DeviceOsdMapping{
 		Entries: map[string]*DeviceOsdIDEntry{
 			"data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}},
@@ -1201,7 +1207,7 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) {
 	clusterInfo := &cephclient.ClusterInfo{
 		CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0},
 	}
-	a := &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1"}
+	a := &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 
 	devices := &DeviceOsdMapping{
 		Entries: map[string]*DeviceOsdIDEntry{
@@ -1233,7 +1239,7 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) {
 	clusterInfo = &cephclient.ClusterInfo{
 		CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0},
 	}
-	a = &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1"}
+	a = &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 
 	devices = &DeviceOsdMapping{
 		Entries: map[string]*DeviceOsdIDEntry{
@@ -1459,7 +1465,7 @@ func TestInitializeBlockWithMD(t *testing.T) {
 
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
-		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 10}}, nodeName: "node1"}
+		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 10}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 		context := &clusterd.Context{
 			Executor: executor,
 			Devices: []*sys.LocalDisk{
@@ -1512,7 +1518,7 @@ func TestInitializeBlockWithMD(t *testing.T) {
 
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
-		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 4}}, nodeName: "node1"}
+		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 4}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
 		context := &clusterd.Context{
 			Executor: executor,
 			Devices: []*sys.LocalDisk{

pkg/operator/ceph/cluster/osd/config/config.go

@@ -18,7 +18,10 @@ limitations under the License.
 package config
 
 import (
+	"fmt"
 	"strconv"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 )
 
 const (
@@ -42,6 +45,19 @@ type StoreConfig struct {
 	DeviceClass     string `json:"deviceClass,omitempty"`
 	InitialWeight   string `json:"initialWeight,omitempty"`
 	PrimaryAffinity string `json:"primaryAffinity,omitempty"`
+	StoreType       string `json:"storeType,omitempty"`
+}
+
+func (s StoreConfig) IsValidStoreType() bool {
+	if s.StoreType == string(cephv1.StoreTypeBlueStore) || s.StoreType == string(cephv1.StoreTypeBlueStoreRDR) {
+		return true
+	}
+
+	return false
+}
+
+func (s StoreConfig) GetStoreFlag() string {
+	return fmt.Sprintf("--%s", s.StoreType)
 }
 
 // NewStoreConfig returns a StoreConfig with proper defaults set.

pkg/operator/ceph/cluster/osd/envs.go

@@ -48,6 +48,7 @@ const (
 	lvBackedPVVarName                   = "ROOK_LV_BACKED_PV"
 	CrushDeviceClassVarName             = "ROOK_OSD_CRUSH_DEVICE_CLASS"
 	CrushInitialWeightVarName           = "ROOK_OSD_CRUSH_INITIAL_WEIGHT"
+	OSDStoreTypeVarName                 = "ROOK_OSD_STORE_TYPE"
 	CrushRootVarName                    = "ROOK_CRUSHMAP_ROOT"
 	tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES"
 )
@@ -80,6 +81,8 @@ func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string, prepa
 				},
 			}},
 		}...)
+
+		envVars = append(envVars, osdStoreTypeEnvVar(c.spec.Storage.GetOSDStore()))
 	}
 
 	// Give a hint to the prepare pod for what the host in the CRUSH map should be
@@ -168,6 +171,10 @@ func crushDeviceClassEnvVar(crushDeviceClass string) v1.EnvVar {
 	return v1.EnvVar{Name: CrushDeviceClassVarName, Value: crushDeviceClass}
 }
 
+func osdStoreTypeEnvVar(storeType string) v1.EnvVar {
+	return v1.EnvVar{Name: OSDStoreTypeVarName, Value: storeType}
+}
+
 func crushInitialWeightEnvVar(crushInitialWeight string) v1.EnvVar {
 	return v1.EnvVar{Name: CrushInitialWeightVarName, Value: crushInitialWeight}
 }

pkg/operator/ceph/cluster/osd/labels.go

@@ -59,6 +59,7 @@ func (c *Cluster) getOSDLabels(osd OSDInfo, failureDomainValue string, portable
 	labels[FailureDomainKey] = failureDomainValue
 	labels[portableKey] = strconv.FormatBool(portable)
 	labels[deviceClass] = osd.DeviceClass
+	labels[osdStore] = c.spec.Storage.GetOSDStore()
 
 	for k, v := range getOSDTopologyLocationLabels(osd.Location) {
 		labels[k] = v

pkg/operator/ceph/cluster/osd/osd.go

@@ -67,6 +67,7 @@ const (
 	bluestorePVCWal                = "wal"
 	bluestorePVCData               = "data"
 	deviceClass                    = "device-class"
+	osdStore                       = "osd-store"
 )
 
 // Cluster keeps track of the OSDs

pkg/operator/ceph/cluster/osd/spec.go

@@ -835,7 +835,6 @@ func (c *Cluster) getActivateOSDInitContainer(configDir, namespace, osdID string
 		walDeviceEnvVar(osdInfo.WalPath),
 		v1.EnvVar{Name: "ROOK_OSD_ID", Value: osdID},
 	)
-	osdStore := "--bluestore"
 
 	// Build empty dir osd path to something like "/var/lib/ceph/osd/ceph-0"
 	activateOSDMountPathID := activateOSDMountPath + osdID
@@ -853,11 +852,13 @@ func (c *Cluster) getActivateOSDInitContainer(configDir, namespace, osdID string
 		volMounts = append(volMounts, getPvcOSDBridgeMount(osdProps.pvc.ClaimName))
 	}
 
+	osdStoreFlag := c.spec.Storage.GetOSDStoreFlag()
+
 	container := &v1.Container{
 		Command: []string{
 			"/bin/bash",
 			"-c",
-			fmt.Sprintf(activateOSDOnNodeCode, c.clusterInfo.FSID, osdInfo.UUID, osdStore, osdInfo.CVMode, blockPathVarName),
+			fmt.Sprintf(activateOSDOnNodeCode, c.clusterInfo.FSID, osdInfo.UUID, osdStoreFlag, osdInfo.CVMode, blockPathVarName),
 		},
 		Name:            "activate",
 		Image:           c.spec.CephVersion.Image,