diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index ddd1fb1f8..c35787ba3 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -8181,6 +8181,36 @@ string +

OSDStore +

+

+(Appears on:StorageScopeSpec) +

+
+

OSDStore is the backend storage type used for creating the OSDs

+
+ + + + + + + + + + + + + +
FieldDescription
+type
+ +string + +		 +(Optional) +

Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used

+

ObjectEndpoints

@@ -11135,8 +11165,40 @@ Selection (Optional) + + +store
+ + +OSDStore + + + + +(Optional) + + +

StoreType +(string alias)

+
+
+ + + + + + + + + + + + +
ValueDescription

"bluestore"

StoreTypeBlueStore is the bluestore backend storage for OSDs

+

"bluestore-rdr"

StoreTypeBlueStoreRDR is the bluestore-rdr backed storage for OSDs

+

StretchClusterSpec

diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 840b6f68c..276c9bc17 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -28,6 +28,7 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/cmd/rook/rook" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" osddaemon "github.com/rook/rook/pkg/daemon/ceph/osd" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" @@ -131,6 +132,7 @@ func addOSDConfigFlags(command *cobra.Command) { command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt") command.Flags().StringVar(&cfg.storeConfig.DeviceClass, "osd-crush-device-class", "", "The device class for all OSDs configured on this node") command.Flags().StringVar(&cfg.storeConfig.InitialWeight, "osd-crush-initial-weight", "", "The initial weight of OSD in TiB units") + command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store-type", string(cephv1.StoreTypeBlueStore), "the osd store type such as bluestore") } func init() { diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 2e5be4d78..7f39bdfbd 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -4360,6 +4360,16 @@ spec: type: object nullable: true type: array + store: + description: OSDStore is the backend storage type used for creating the OSDs + properties: + type: + description: Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used + enum: + - bluestore + - bluestore-rdr + type: string + type: object useAllDevices: description: Whether to consume all the storage devices found on a machine type: boolean diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 7422ee6a9..c2c1d7f75 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -4358,6 +4358,16 @@ spec: type: object nullable: true type: array + store: + description: OSDStore is the backend storage type used for creating the OSDs + properties: + type: + description: Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used + enum: + - bluestore + - bluestore-rdr + type: string + type: object useAllDevices: description: Whether to consume all the storage devices found on a machine type: boolean diff --git a/pkg/apis/ceph.rook.io/v1/storage.go b/pkg/apis/ceph.rook.io/v1/storage.go index 1f5cd9c71..36a66ebdb 100644 --- a/pkg/apis/ceph.rook.io/v1/storage.go +++ b/pkg/apis/ceph.rook.io/v1/storage.go @@ -15,6 +15,18 @@ limitations under the License. */ package v1 +import "fmt" + +type StoreType string + +const ( + // StoreTypeBlueStore is the bluestore backend storage for OSDs + StoreTypeBlueStore StoreType = "bluestore" + + // StoreTypeBlueStoreRDR is the bluestore-rdr backed storage for OSDs + StoreTypeBlueStoreRDR StoreType = "bluestore-rdr" +) + // AnyUseAllDevices gets whether to use all devices func (s *StorageScopeSpec) AnyUseAllDevices() bool { if s.Selection.GetUseAllDevices() { @@ -168,3 +180,19 @@ func (s *StorageScopeSpec) IsOnPVCEncrypted() bool { return false } + +// GetOSDStore returns osd backend store type provided in the cluster spec +func (s *StorageScopeSpec) GetOSDStore() string { + if s.Store.Type == "" { + return string(StoreTypeBlueStore) + } + return s.Store.Type +} + +// GetOSDStoreFlag returns osd backend store type prefixed with "--" +func (s *StorageScopeSpec) GetOSDStoreFlag() string { + if s.Store.Type == "" { + return fmt.Sprintf("--%s", StoreTypeBlueStore) + } + return fmt.Sprintf("--%s", s.Store.Type) +} diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 8f762ccd9..c6a496309 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -2587,6 +2587,16 @@ type StorageScopeSpec struct { // +nullable // +optional StorageClassDeviceSets []StorageClassDeviceSet `json:"storageClassDeviceSets,omitempty"` + // +optional + Store OSDStore `json:"store,omitempty"` +} + +// OSDStore is the backend storage type used for creating the OSDs +type OSDStore struct { + // Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used + // +optional + // +kubebuilder:validation:Enum=bluestore;bluestore-rdr; + Type string `json:"type,omitempty"` } // Node is a storage nodes diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 042e278d9..7be6f7f05 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -3197,6 +3197,22 @@ func (in *NotificationKeyFilterRule) DeepCopy() *NotificationKeyFilterRule { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OSDStore) DeepCopyInto(out *OSDStore) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OSDStore. +func (in *OSDStore) DeepCopy() *OSDStore { + if in == nil { + return nil + } + out := new(OSDStore) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectEndpoints) DeepCopyInto(out *ObjectEndpoints) { *out = *in @@ -4217,6 +4233,7 @@ func (in *StorageScopeSpec) DeepCopyInto(out *StorageScopeSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.Store = in.Store return } diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index a0585e873..440115bfa 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -29,6 +29,7 @@ import ( "strings" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" @@ -220,6 +221,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device // we need to return the block if raw mode is used and the lv if lvm mode baseCommand := "stdbuf" var baseArgs []string + storeFlag := a.storeConfig.GetStoreFlag() // Create a specific log directory so that each prepare command will have its own log // Only do this if nothing is present so that we don't override existing logs @@ -227,10 +229,9 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device err := os.MkdirAll(cvLogDir, 0750) if err != nil { logger.Errorf("failed to create ceph-volume log directory %q, continue with default %q. %v", cvLogDir, cephLogDir, err) - baseArgs = []string{"-oL", cephVolumeCmd, "raw", "prepare", "--bluestore"} + baseArgs = []string{"-oL", cephVolumeCmd, "raw", "prepare", storeFlag} } else { - // Always force Bluestore! - baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir, "raw", "prepare", "--bluestore"} + baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir, "raw", "prepare", storeFlag} } var metadataArg, walArg []string @@ -511,7 +512,9 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO func (a *OsdAgent) initializeDevicesRawMode(context *clusterd.Context, devices *DeviceOsdMapping) error { baseCommand := "stdbuf" cephVolumeMode := "raw" - baseArgs := []string{"-oL", cephVolumeCmd, cephVolumeMode, "prepare", "--bluestore"} + storeFlag := a.storeConfig.GetStoreFlag() + + baseArgs := []string{"-oL", cephVolumeCmd, cephVolumeMode, "prepare", storeFlag} for name, device := range devices.Entries { deviceArg := path.Join("/dev", name) @@ -550,8 +553,7 @@ func (a *OsdAgent) initializeDevicesRawMode(context *clusterd.Context, devices * } func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *DeviceOsdMapping) error { - storeFlag := "--bluestore" - + storeFlag := a.storeConfig.GetStoreFlag() logPath := "/tmp/ceph-log" if err := os.MkdirAll(logPath, 0700); err != nil { return errors.Wrapf(err, "failed to create dir %q", logPath) @@ -628,7 +630,7 @@ func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices * metadataDevices[md]["devices"] = deviceArg } deviceDBSizeMB := getDatabaseSize(a.storeConfig.DatabaseSizeMB, device.Config.DatabaseSizeMB) - if storeFlag == "--bluestore" && deviceDBSizeMB > 0 { + if a.storeConfig.IsValidStoreType() && deviceDBSizeMB > 0 { if deviceDBSizeMB < cephVolumeMinDBSize { // ceph-volume will convert this value to ?G. It needs to be > 1G to invoke lvcreate. logger.Infof("skipping databaseSizeMB setting (%d). For it should be larger than %dMB.", deviceDBSizeMB, cephVolumeMinDBSize) @@ -858,6 +860,11 @@ func GetCephVolumeLVMOSDs(context *clusterd.Context, clusterInfo *client.Cluster lvPath = lv } + osdStore := os.Getenv(oposd.OSDStoreTypeVarName) + if osdStore == "" { + osdStore = string(cephv1.StoreTypeBlueStore) + } + osd := oposd.OSDInfo{ ID: id, Cluster: "ceph", @@ -866,7 +873,7 @@ func GetCephVolumeLVMOSDs(context *clusterd.Context, clusterInfo *client.Cluster SkipLVRelease: skipLVRelease, LVBackedPV: lvBackedPV, CVMode: cvMode, - Store: "bluestore", + Store: osdStore, DeviceClass: osdDeviceClass, } osds = append(osds, osd) @@ -1046,6 +1053,11 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster blockPath = block } + osdStore := os.Getenv(oposd.OSDStoreTypeVarName) + if osdStore == "" { + osdStore = string(cephv1.StoreTypeBlueStore) + } + osd := oposd.OSDInfo{ ID: osdID, Cluster: "ceph", @@ -1061,7 +1073,7 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster SkipLVRelease: true, LVBackedPV: lvBackedPV, CVMode: cvMode, - Store: "bluestore", + Store: osdStore, Encrypted: strings.Contains(blockPath, "-dmcrypt"), } diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 840368841..03fe7b47f 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -328,6 +328,8 @@ func TestConfigureCVDevices(t *testing.T) { cephConfigDir = originalCephConfigDir }() + os.Setenv(oposd.OSDStoreTypeVarName, "bluestore") + nodeName := "set1-data-0-rpf2k" mountedDev := "/mnt/" + nodeName mapperDev := "/dev/mapper/rook--vg-rook--lv1" @@ -385,7 +387,7 @@ func TestConfigureCVDevices(t *testing.T) { Context: context.TODO(), } context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir} - agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{DeviceClass: "myds"}} + agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{DeviceClass: "myds", StoreType: "bluestore"}} devices := createPVCAvailableDevices() deviceOSDs, err := agent.configureCVDevices(context, devices) assert.Nil(t, err) @@ -438,7 +440,7 @@ func TestConfigureCVDevices(t *testing.T) { CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0}, // It supports raw mode OSD FSID: clusterFSID, } - agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true} + agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{StoreType: "bluestore"}} devices := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}} deviceOSDs, err := agent.configureCVDevices(context, devices) @@ -509,7 +511,7 @@ func TestConfigureCVDevices(t *testing.T) { CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0}, // It supports raw mode OSD FSID: clusterFSID, } - agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true} + agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{StoreType: "bluestore"}} devices := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}} deviceOSDs, err := agent.configureCVDevices(context, devices) @@ -562,7 +564,7 @@ func TestConfigureCVDevices(t *testing.T) { CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 1}, // It supports raw mode OSD FSID: clusterFSID, } - agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, storeConfig: config.StoreConfig{DeviceClass: "myclass"}} + agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, storeConfig: config.StoreConfig{DeviceClass: "myclass", StoreType: "bluestore"}} devices := &DeviceOsdMapping{ Entries: map[string]*DeviceOsdIDEntry{ "vdb1": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/dev/vdb1"}, DeviceInfo: &sys.LocalDisk{Type: sys.PartType}}, @@ -632,6 +634,7 @@ func testBaseArgs(args []string) error { } func TestInitializeBlock(t *testing.T) { + os.Setenv(oposd.OSDStoreTypeVarName, "bluestore") // Common vars for all the tests devices := &DeviceOsdMapping{ Entries: map[string]*DeviceOsdIDEntry{ @@ -663,7 +666,7 @@ func TestInitializeBlock(t *testing.T) { return errors.Errorf("unknown command %s %s", command, args) } - a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1"} + a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} context := &clusterd.Context{Executor: executor} err := a.initializeDevicesLVMMode(context, devices) @@ -695,7 +698,7 @@ func TestInitializeBlock(t *testing.T) { return errors.Errorf("unknown command %s %s", command, args) } - a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{EncryptedDevice: true}} + a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{EncryptedDevice: true, StoreType: "bluestore"}} context := &clusterd.Context{Executor: executor} err := a.initializeDevicesLVMMode(context, devices) @@ -727,7 +730,7 @@ func TestInitializeBlock(t *testing.T) { return errors.Errorf("unknown command %s %s", command, args) } - a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{OSDsPerDevice: 3}} + a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{OSDsPerDevice: 3, StoreType: "bluestore"}} context := &clusterd.Context{Executor: executor} err := a.initializeDevicesLVMMode(context, devices) @@ -759,7 +762,7 @@ func TestInitializeBlock(t *testing.T) { return errors.Errorf("unknown command %s %s", command, args) } - a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{DeviceClass: "hybrid"}} + a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{DeviceClass: "hybrid", StoreType: "bluestore"}} context := &clusterd.Context{Executor: executor} err := a.initializeDevicesLVMMode(context, devices) assert.NoError(t, err, "failed crush device class test") @@ -813,7 +816,7 @@ func TestInitializeBlock(t *testing.T) { return "", errors.Errorf("unknown command %s %s", command, args) } - a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1"} + a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} context := &clusterd.Context{ Executor: executor, Devices: []*sys.LocalDisk{ @@ -878,7 +881,7 @@ func TestInitializeBlock(t *testing.T) { return "", errors.Errorf("unknown command %s %s", command, args) } - a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1"} + a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} context := &clusterd.Context{ Executor: executor, Devices: []*sys.LocalDisk{ @@ -962,7 +965,8 @@ func TestInitializeBlock(t *testing.T) { clusterInfo: &cephclient.ClusterInfo{ CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}, }, - nodeName: "node1", + nodeName: "node1", + storeConfig: config.StoreConfig{StoreType: "bluestore"}, } context := &clusterd.Context{Executor: executor, Devices: []*sys.LocalDisk{ @@ -1043,7 +1047,8 @@ func TestInitializeBlock(t *testing.T) { clusterInfo: &cephclient.ClusterInfo{ CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}, }, - nodeName: "node1", + nodeName: "node1", + storeConfig: config.StoreConfig{StoreType: "bluestore"}, } context := &clusterd.Context{Executor: executor, Devices: []*sys.LocalDisk{ @@ -1086,7 +1091,8 @@ func TestInitializeBlock(t *testing.T) { clusterInfo: &cephclient.ClusterInfo{ CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}, }, - nodeName: "node1", + nodeName: "node1", + storeConfig: config.StoreConfig{StoreType: "bluestore"}, } context := &clusterd.Context{Executor: executor, Devices: []*sys.LocalDisk{ @@ -1126,7 +1132,7 @@ func TestInitializeBlockPVC(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{ CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0}, } - a := &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1"} + a := &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} devices := &DeviceOsdMapping{ Entries: map[string]*DeviceOsdIDEntry{ "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, @@ -1158,7 +1164,7 @@ func TestInitializeBlockPVC(t *testing.T) { clusterInfo = &cephclient.ClusterInfo{ CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0}, } - a = &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1"} + a = &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} devices = &DeviceOsdMapping{ Entries: map[string]*DeviceOsdIDEntry{ "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, @@ -1201,7 +1207,7 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{ CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0}, } - a := &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1"} + a := &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} devices := &DeviceOsdMapping{ Entries: map[string]*DeviceOsdIDEntry{ @@ -1233,7 +1239,7 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) { clusterInfo = &cephclient.ClusterInfo{ CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 0}, } - a = &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1"} + a = &OsdAgent{clusterInfo: clusterInfo, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} devices = &DeviceOsdMapping{ Entries: map[string]*DeviceOsdIDEntry{ @@ -1459,7 +1465,7 @@ func TestInitializeBlockWithMD(t *testing.T) { return "", errors.Errorf("unknown command %s %s", command, args) } - a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 10}}, nodeName: "node1"} + a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 10}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} context := &clusterd.Context{ Executor: executor, Devices: []*sys.LocalDisk{ @@ -1512,7 +1518,7 @@ func TestInitializeBlockWithMD(t *testing.T) { return "", errors.Errorf("unknown command %s %s", command, args) } - a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 4}}, nodeName: "node1"} + a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 4}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}} context := &clusterd.Context{ Executor: executor, Devices: []*sys.LocalDisk{ diff --git a/pkg/operator/ceph/cluster/osd/config/config.go b/pkg/operator/ceph/cluster/osd/config/config.go index 8b2f0bc1d..b9a870cd9 100644 --- a/pkg/operator/ceph/cluster/osd/config/config.go +++ b/pkg/operator/ceph/cluster/osd/config/config.go @@ -18,7 +18,10 @@ limitations under the License. package config import ( + "fmt" "strconv" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" ) const ( @@ -42,6 +45,19 @@ type StoreConfig struct { DeviceClass string `json:"deviceClass,omitempty"` InitialWeight string `json:"initialWeight,omitempty"` PrimaryAffinity string `json:"primaryAffinity,omitempty"` + StoreType string `json:"storeType,omitempty"` +} + +func (s StoreConfig) IsValidStoreType() bool { + if s.StoreType == string(cephv1.StoreTypeBlueStore) || s.StoreType == string(cephv1.StoreTypeBlueStoreRDR) { + return true + } + + return false +} + +func (s StoreConfig) GetStoreFlag() string { + return fmt.Sprintf("--%s", s.StoreType) } // NewStoreConfig returns a StoreConfig with proper defaults set. diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index 7ad7b2b77..2ffc1ca10 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -48,6 +48,7 @@ const ( lvBackedPVVarName = "ROOK_LV_BACKED_PV" CrushDeviceClassVarName = "ROOK_OSD_CRUSH_DEVICE_CLASS" CrushInitialWeightVarName = "ROOK_OSD_CRUSH_INITIAL_WEIGHT" + OSDStoreTypeVarName = "ROOK_OSD_STORE_TYPE" CrushRootVarName = "ROOK_CRUSHMAP_ROOT" tcmallocMaxTotalThreadCacheBytesEnv = "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES" ) @@ -80,6 +81,8 @@ func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string, prepa }, }}, }...) + + envVars = append(envVars, osdStoreTypeEnvVar(c.spec.Storage.GetOSDStore())) } // Give a hint to the prepare pod for what the host in the CRUSH map should be @@ -168,6 +171,10 @@ func crushDeviceClassEnvVar(crushDeviceClass string) v1.EnvVar { return v1.EnvVar{Name: CrushDeviceClassVarName, Value: crushDeviceClass} } +func osdStoreTypeEnvVar(storeType string) v1.EnvVar { + return v1.EnvVar{Name: OSDStoreTypeVarName, Value: storeType} +} + func crushInitialWeightEnvVar(crushInitialWeight string) v1.EnvVar { return v1.EnvVar{Name: CrushInitialWeightVarName, Value: crushInitialWeight} } diff --git a/pkg/operator/ceph/cluster/osd/labels.go b/pkg/operator/ceph/cluster/osd/labels.go index 08a72989e..b3e7e6471 100644 --- a/pkg/operator/ceph/cluster/osd/labels.go +++ b/pkg/operator/ceph/cluster/osd/labels.go @@ -59,6 +59,7 @@ func (c *Cluster) getOSDLabels(osd OSDInfo, failureDomainValue string, portable labels[FailureDomainKey] = failureDomainValue labels[portableKey] = strconv.FormatBool(portable) labels[deviceClass] = osd.DeviceClass + labels[osdStore] = c.spec.Storage.GetOSDStore() for k, v := range getOSDTopologyLocationLabels(osd.Location) { labels[k] = v diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index efaf14492..bd8a62ade 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -67,6 +67,7 @@ const ( bluestorePVCWal = "wal" bluestorePVCData = "data" deviceClass = "device-class" + osdStore = "osd-store" ) // Cluster keeps track of the OSDs diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 27e55585a..3097c3185 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -835,7 +835,6 @@ func (c *Cluster) getActivateOSDInitContainer(configDir, namespace, osdID string walDeviceEnvVar(osdInfo.WalPath), v1.EnvVar{Name: "ROOK_OSD_ID", Value: osdID}, ) - osdStore := "--bluestore" // Build empty dir osd path to something like "/var/lib/ceph/osd/ceph-0" activateOSDMountPathID := activateOSDMountPath + osdID @@ -853,11 +852,13 @@ func (c *Cluster) getActivateOSDInitContainer(configDir, namespace, osdID string volMounts = append(volMounts, getPvcOSDBridgeMount(osdProps.pvc.ClaimName)) } + osdStoreFlag := c.spec.Storage.GetOSDStoreFlag() + container := &v1.Container{ Command: []string{ "/bin/bash", "-c", - fmt.Sprintf(activateOSDOnNodeCode, c.clusterInfo.FSID, osdInfo.UUID, osdStore, osdInfo.CVMode, blockPathVarName), + fmt.Sprintf(activateOSDOnNodeCode, c.clusterInfo.FSID, osdInfo.UUID, osdStoreFlag, osdInfo.CVMode, blockPathVarName), }, Name: "activate", Image: c.spec.CephVersion.Image,