ceph: add raw mode for non-pvc osd

For simple OSD scenario that don't require the following:

* an encrypted osd
* an osd with a backing device
* multiple osd on the same block device

The OSD will be prepared using the ceph-volume raw mode which does
not use LVM on non-pvc case.

Closes: https://github.com/rook/rook/issues/4768
Signed-off-by: Sébastien Han <seb@redhat.com>

PendingReleaseNotes.md

@@ -26,3 +26,4 @@ v1.6...
 * Extending the support of vault KMS configuration for Ceph RGW
 * Enable disruption budgets (PDBs) by default for Mon, RGW, MDS, and OSD daemons
 * Add CephFilesystemMirror CRD to deploy cephfs-mirror daemon
+* Ceph OSD: as of Nautilus 14.2.14 and Octopus 15.2.9 if the OSD scenario is simple (one OSD per disk) we won't use LVM to prepare the disk anymore

pkg/daemon/ceph/osd/daemon.go

@@ -196,17 +196,8 @@ func writeCephConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterI
 
 // Provision provisions an OSD
 func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity string) error {
-
-	// Check for the presence of LVM on the host when NOT running on PVC
-	// since this scenario is still using LVM
-	if !agent.pvcBacked {
-		ne := NewNsenter(context, lvmCommandToCheck, []string{"--help"})
-		err := ne.checkIfBinaryExistsOnHost()
-		if err != nil {
-			return errors.Wrapf(err, "binary %q does not exist on the host, make sure lvm2 package is installed", lvmCommandToCheck)
-		}
+	if agent.pvcBacked {
 		// Init KMS store, retrieve the KEK and store it as an env var for ceph-volume
-	} else {
 		err := setKEKinEnv(context, agent.clusterInfo)
 		if err != nil {
 			return errors.Wrap(err, "failed to set kek as an environment variable")

pkg/daemon/ceph/osd/volume.go

@@ -27,7 +27,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
@@ -57,8 +56,12 @@ var (
 	cvLogDir      = ""
 	// The "ceph-volume raw" command is available since Ceph 14.2.8 as well as partition support in ceph-volume
 	cephVolumeRawModeMinCephVersion = cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}
-	isEncrypted                     = os.Getenv(oposd.EncryptedDeviceEnvVarName) == "true"
-	isOnPVC                         = os.Getenv(osd.PVCBackedOSDVarName) == "true"
+	// The Ceph Nautilus to include a retry to acquire device lock
+	cephFlockFixNautilusMinCephVersion = cephver.CephVersion{Major: 14, Minor: 2, Extra: 14}
+	// The Ceph Octopus to include a retry to acquire device lock
+	cephFlockFixOctopusMinCephVersion = cephver.CephVersion{Major: 15, Minor: 2, Extra: 9}
+	isEncrypted                       = os.Getenv(oposd.EncryptedDeviceEnvVarName) == "true"
+	isOnPVC                           = os.Getenv(osd.PVCBackedOSDVarName) == "true"
 )
 
 type osdInfoBlock struct {
@@ -179,6 +182,13 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 		}
 		osds = append(osds, lvmOsds...)
 
+		// List existing OSD(s) configured with ceph-volume raw mode
+		rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, "", "", false)
+		if err != nil {
+			logger.Infof("failed to get device already provisioned by ceph-volume raw. %v", err)
+		}
+		osds = append(osds, rawOsds...)
+
 		return osds, nil
 	}
 
@@ -200,11 +210,17 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 		}
 	}
 
-	// Update LVM configuration file
-	// Only do this after Ceph Nautilus 14.2.6 since it will use the ceph-volume raw mode by default and not LVM anymore
-	if !a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) {
-		if err := UpdateLVMConfig(context, a.pvcBacked, lvBackedPV); err != nil {
-			return nil, errors.Wrap(err, "failed to update lvm configuration file")
+	// Should we use ceph-volume raw mode?
+	useRawMode, err := a.useRawMode(context, a.pvcBacked)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to determine which ceph-volume mode to use")
+	}
+
+	// If not raw mode we must execute a few LVM prerequisites
+	if !useRawMode {
+		err = lvmPreReq(context, a.pvcBacked, lvBackedPV)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to run lvm prerequisites")
 		}
 	}
 
@@ -214,31 +230,45 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 			return nil, errors.Wrap(err, "failed to initialize devices on PVC")
 		}
 	} else {
-		if err = a.initializeDevices(context, devices); err != nil {
-			return nil, errors.Wrap(err, "failed to initialize devices")
+		// Initialize block device OSD without LVM
+		if useRawMode {
+			logger.Info("initializing osd disk with raw mode")
+			err := a.initializeDevicesRawMode(context, devices)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to initialize raw based osd")
+			}
+		} else {
+			// Initialize block device OSD with LVM
+			logger.Info("initializing osd disk with lvm mode")
+			err := a.initializeDevicesLVMMode(context, devices)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to initialize lvm based osd")
+			}
 		}
 	}
 
 	// List OSD configured with ceph-volume lvm mode
+	// List THE configured OSD with ceph-volume raw mode
+	// When the block is encrypted we need to list against the encrypted device mapper
+	if !isEncrypted {
+		block = fmt.Sprintf("/mnt/%s", a.nodeName)
+	}
+	// List ALL OSDs when not running on PVC
+	if !a.pvcBacked {
+		block = ""
+	}
+	rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, walBlock, lvBackedPV)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get devices already provisioned by ceph-volume raw")
+	}
+	osds = append(osds, rawOsds...)
+
 	lvmOsds, err = GetCephVolumeLVMOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, false, lvBackedPV)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get devices already provisioned by ceph-volume lvm")
 	}
 	osds = append(osds, lvmOsds...)
 
-	// List THE configured OSD with ceph-volume raw mode
-	if a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) {
-		// When the block is encrypted we need to list against the encrypted device mapper
-		if !isEncrypted {
-			block = fmt.Sprintf("/mnt/%s", a.nodeName)
-		}
-		rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, walBlock, lvBackedPV)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to get devices already provisioned by ceph-volume raw")
-		}
-		osds = append(osds, rawOsds...)
-	}
-
 	return osds, err
 }
 
@@ -461,13 +491,27 @@ func UpdateLVMConfig(context *clusterd.Context, onPVC, lvBackedPV bool) error {
 	return nil
 }
 
-func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceOsdMapping) error {
-	// We need at least Ceph 14.2.8 to use ceph-volume raw mode
-	useRawMode := a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion)
+func (a *OsdAgent) useRawMode(context *clusterd.Context, pvcBacked bool) (bool, error) {
+	if pvcBacked {
+		return a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion), nil
+	}
+
+	var useRawMode bool
+	// Can we safely use ceph-volume raw mode in the non-PVC case?
+	// On non-PVC we see a race between systemd-udev and the osd process to acquire the lock on the device
+	if a.clusterInfo.CephVersion.IsNautilus() && a.clusterInfo.CephVersion.IsAtLeast(cephFlockFixNautilusMinCephVersion) {
+		logger.Debugf("will use raw mode since cluster version is at least %v", cephFlockFixNautilusMinCephVersion)
+		useRawMode = true
+	}
+
+	if a.clusterInfo.CephVersion.IsOctopus() && a.clusterInfo.CephVersion.IsAtLeast(cephFlockFixOctopusMinCephVersion) {
+		logger.Debugf("will use raw mode since cluster version is at least %v", cephFlockFixOctopusMinCephVersion)
+		useRawMode = true
+	}
 
 	// ceph-volume raw mode does not support encryption yet
 	if a.storeConfig.EncryptedDevice {
-		logger.Infof("won't use raw mode since encryption is enabled")
+		logger.Debug("won't use raw mode since encryption is enabled")
 		useRawMode = false
 	}
 
@@ -475,42 +519,23 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
 	osdsPerDeviceCountString := sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice)
 	osdsPerDeviceCount, err := strconv.Atoi(osdsPerDeviceCountString)
 	if err != nil {
-		return errors.Wrapf(err, "failed to convert string %q to integer", osdsPerDeviceCountString)
+		return false, errors.Wrapf(err, "failed to convert string %q to integer", osdsPerDeviceCountString)
 	}
 	if osdsPerDeviceCount > 1 {
-		logger.Infof("won't use raw mode since osd per device > 1 is enabled %d", osdsPerDeviceCount)
+		logger.Debugf("won't use raw mode since osd per device is %d", osdsPerDeviceCount)
 		useRawMode = false
 	}
 
 	// ceph-volume raw mode mode does not support metadata device if not running on PVC because the user has specified a whole device
 	if a.metadataDevice != "" {
-		logger.Infof("won't use raw mode since there is a metadata device %q", a.metadataDevice)
+		logger.Debugf("won't use raw mode since there is a metadata device %q", a.metadataDevice)
 		useRawMode = false
 	}
 
-	// Initialize block device OSD without LVM
-	if useRawMode {
-		logger.Info("Initializing OSD disk with raw mode")
-		err := a.initializeDevicesRawMode(context, devices)
-		if err != nil {
-			return errors.Wrap(err, "failed to initialize raw based osd")
-		}
-	} else {
-		// Initialize block device OSD with LVM
-		logger.Info("Initializing OSD disk with LVM mode")
-		err := a.initializeDevicesLVMMode(context, devices)
-		if err != nil {
-			return errors.Wrap(err, "failed to initialize lvm based osd")
-		}
-	}
-
-	return nil
+	return useRawMode, nil
 }
 
 func (a *OsdAgent) initializeDevicesRawMode(context *clusterd.Context, devices *DeviceOsdMapping) error {
-
-	time.Sleep(time.Hour * 350)
-
 	baseCommand := "stdbuf"
 	cephVolumeMode := "raw"
 	baseArgs := []string{"-oL", cephVolumeCmd, cephVolumeMode, "prepare", "--bluestore"}
@@ -546,7 +571,6 @@ func (a *OsdAgent) initializeDevicesRawMode(context *clusterd.Context, devices *
 	}
 
 	return nil
-
 }
 
 func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *DeviceOsdMapping) error {
@@ -768,6 +792,24 @@ func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *
 	return nil
 }
 
+func lvmPreReq(context *clusterd.Context, pvcBacked, lvBackedPV bool) error {
+	// Check for the presence of LVM on the host when NOT running on PVC
+	// since this scenario is still using LVM
+	ne := NewNsenter(context, lvmCommandToCheck, []string{"--help"})
+	err := ne.checkIfBinaryExistsOnHost()
+	if err != nil {
+		return errors.Wrapf(err, "binary %q does not exist on the host, make sure lvm2 package is installed", lvmCommandToCheck)
+	}
+
+	// Update LVM configuration file
+	// Only do this after Ceph Nautilus 14.2.6 since it will use the ceph-volume raw mode by default and not LVM anymore
+	if err := UpdateLVMConfig(context, pvcBacked, lvBackedPV); err != nil {
+		return errors.Wrap(err, "failed to update lvm configuration file")
+	}
+
+	return nil
+}
+
 func getDatabaseSize(globalSize int, deviceSize int) int {
 	if deviceSize > 0 {
 		globalSize = deviceSize
@@ -890,8 +932,6 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
 	if block == "" {
 		setDevicePathFromList = true
 		args = []string{cvMode, "list", "--format", "json"}
-	} else {
-		blockPath = block
 	}
 
 	result, err := callCephVolume(context, false, args...)

pkg/daemon/ceph/osd/volume_test.go

@@ -28,6 +28,7 @@ import (
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
+	"github.com/rook/rook/pkg/operator/ceph/version"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
@@ -339,10 +340,10 @@ func TestConfigureCVDevices(t *testing.T) {
 			if command == "lsblk" && args[0] == mountedDev {
 				return fmt.Sprintf(`SIZE="17179869184" ROTA="1" RO="0" TYPE="lvm" PKNAME="" NAME="%s" KNAME="/dev/dm-1, a ...interface{})`, mapperDev), nil
 			}
-			if args[1] == "ceph-volume" && args[4] == "lvm" && args[5] == "list" && args[6] == mapperDev {
+			if contains(args, "lvm") && contains(args, "list") {
 				return `{}`, nil
 			}
-			if args[1] == "ceph-volume" && args[4] == "raw" && args[5] == "list" && args[6] == mountedDev {
+			if contains(args, "raw") && contains(args, "list") {
 				return fmt.Sprintf(`{
 				"0": {
 					"ceph_fsid": "%s",
@@ -361,6 +362,9 @@ func TestConfigureCVDevices(t *testing.T) {
 			if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[6] == mapperDev {
 				return initializeBlockPVCTestResult, nil
 			}
+			if contains(args, "lvm") && contains(args, "list") {
+				return `{}`, nil
+			}
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
 
@@ -397,6 +401,9 @@ func TestConfigureCVDevices(t *testing.T) {
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
 		executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
+			if command == "nsenter" {
+				return "", nil
+			}
 			logger.Infof("[MockExecuteCommandWithCombinedOutput] %s %v", command, args)
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
@@ -579,7 +586,7 @@ func TestInitializeBlock(t *testing.T) {
 		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1"}
 		context := &clusterd.Context{Executor: executor}
 
-		err := a.initializeDevices(context, devices)
+		err := a.initializeDevicesLVMMode(context, devices)
 		assert.NoError(t, err, "failed default behavior test")
 		logger.Info("success, go to next test")
 	}
@@ -611,7 +618,7 @@ func TestInitializeBlock(t *testing.T) {
 		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1", storeConfig: config.StoreConfig{EncryptedDevice: true}}
 		context := &clusterd.Context{Executor: executor}
 
-		err := a.initializeDevices(context, devices)
+		err := a.initializeDevicesLVMMode(context, devices)
 		assert.NoError(t, err, "failed encryption test")
 		logger.Info("success, go to next test")
 	}
@@ -643,7 +650,7 @@ func TestInitializeBlock(t *testing.T) {
 		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1", storeConfig: config.StoreConfig{OSDsPerDevice: 3}}
 		context := &clusterd.Context{Executor: executor}
 
-		err := a.initializeDevices(context, devices)
+		err := a.initializeDevicesLVMMode(context, devices)
 		assert.NoError(t, err, "failed multiple osd test")
 		logger.Info("success, go to next test")
 	}
@@ -674,7 +681,7 @@ func TestInitializeBlock(t *testing.T) {
 		}
 		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1", storeConfig: config.StoreConfig{DeviceClass: "hybrid"}}
 		context := &clusterd.Context{Executor: executor}
-		err := a.initializeDevices(context, devices)
+		err := a.initializeDevicesLVMMode(context, devices)
 		assert.NoError(t, err, "failed crush device class test")
 		logger.Info("success, go to next test")
 	}
@@ -729,7 +736,7 @@ func TestInitializeBlock(t *testing.T) {
 		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1"}
 		context := &clusterd.Context{Executor: executor}
 
-		err := a.initializeDevices(context, devices)
+		err := a.initializeDevicesLVMMode(context, devices)
 		assert.NoError(t, err, "failed metadata test")
 		logger.Info("success, go to next test")
 	}
@@ -784,7 +791,7 @@ func TestInitializeBlock(t *testing.T) {
 		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1"}
 		context := &clusterd.Context{Executor: executor}
 
-		err := a.initializeDevices(context, devices)
+		err := a.initializeDevicesLVMMode(context, devices)
 		assert.NoError(t, err, "failed metadata device by-id test")
 		logger.Info("success, go to next test")
 	}
@@ -1241,7 +1248,67 @@ func TestInitializeBlockWithMD(t *testing.T) {
 		a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 15}}, nodeName: "node1"}
 		context := &clusterd.Context{Executor: executor}
 
-		err := a.initializeDevices(context, devices)
+		err := a.initializeDevicesLVMMode(context, devices)
 		assert.NoError(t, err, "failed default behavior test")
 	}
 }
+
+func TestUseRawMode(t *testing.T) {
+	type fields struct {
+		clusterInfo    *cephclient.ClusterInfo
+		metadataDevice string
+		storeConfig    config.StoreConfig
+		pvcBacked      bool
+	}
+	type args struct {
+		context   *clusterd.Context
+		pvcBacked bool
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    bool
+		wantErr bool
+	}{
+		{"on pvc with lvm", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 5}}, "", config.StoreConfig{}, true}, args{&clusterd.Context{}, true}, false, false},
+		{"on pvc with raw", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 8}}, "", config.StoreConfig{}, true}, args{&clusterd.Context{}, true}, true, false},
+		{"non-pvc with lvm nautilus", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 13}}, "", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, false, false},
+		{"non-pvc with lvm octopus", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 8}}, "", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, false, false},
+		{"non-pvc with raw nautilus simple scenario supported", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 14}}, "", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, true, false},
+		{"non-pvc with raw octopus simple scenario supported", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 9}}, "", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, true, false},
+		{"non-pvc with lvm nautilus complex scenario not supported: encrypted", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 14}}, "", config.StoreConfig{EncryptedDevice: true}, false}, args{&clusterd.Context{}, false}, false, false},
+		{"non-pvc with lvm octopus complex scenario not supported: encrypted", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 9}}, "", config.StoreConfig{EncryptedDevice: true}, false}, args{&clusterd.Context{}, false}, false, false},
+		{"non-pvc with lvm nautilus complex scenario not supported: osd per device > 1", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 14}}, "", config.StoreConfig{OSDsPerDevice: 2}, false}, args{&clusterd.Context{}, false}, false, false},
+		{"non-pvc with lvm octopus complex scenario not supported: osd per device > 1", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 9}}, "", config.StoreConfig{OSDsPerDevice: 2}, false}, args{&clusterd.Context{}, false}, false, false},
+		{"non-pvc with lvm nautilus complex scenario not supported: metadata dev", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 14}}, "/dev/sdb", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, false, false},
+		{"non-pvc with lvm octopus complex scenario not supported: metadata dev", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 9}}, "/dev/sdb", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, false, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := &OsdAgent{
+				clusterInfo:    tt.fields.clusterInfo,
+				metadataDevice: tt.fields.metadataDevice,
+				storeConfig:    tt.fields.storeConfig,
+				pvcBacked:      tt.fields.pvcBacked,
+			}
+			got, err := a.useRawMode(tt.args.context, tt.args.pvcBacked)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("OsdAgent.useRawMode() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("OsdAgent.useRawMode() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func contains(arr []string, str string) bool {
+	for _, a := range arr {
+		if a == str {
+			return true
+		}
+	}
+	return false
+}

pkg/operator/ceph/cluster/osd/spec.go

@@ -296,8 +296,8 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 
 	dataDir := k8sutil.DataDir
 	// Create volume config for /dev so the pod can access devices on the host
-	// Only valid when running OSD with LVM mode
-	if osd.CVMode == "lvm" {
+	// Only valid when running OSD with LVM and Raw mode
+	if !osdProps.onPVC() {
 		devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}
 		volumes = append(volumes, devVolume)
 		devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"}

pkg/operator/ceph/cluster/osd/spec_test.go

@@ -198,7 +198,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
 	blkInitCont := deployment.Spec.Template.Spec.InitContainers[2]
 	assert.Equal(t, 1, len(blkInitCont.VolumeDevices))
 	cont = deployment.Spec.Template.Spec.Containers[0]
-	assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts)
+	assert.Equal(t, 7, len(cont.VolumeMounts), cont.VolumeMounts)
 
 	// Test OSD on PVC with RAW
 	osd = OSDInfo{