From 3dde2609fa0bae87f0a83c629d3ffa7c20caa84c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Mon, 17 Feb 2020 16:36:52 +0100 Subject: [PATCH] ceph: add raw mode for non-pvc osd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For simple OSD scenario that don't require the following: * an encrypted osd * an osd with a backing device * multiple osd on the same block device The OSD will be prepared using the ceph-volume raw mode which does not use LVM on non-pvc case. Closes: https://github.com/rook/rook/issues/4768 Signed-off-by: Sébastien Han --- PendingReleaseNotes.md | 1 + pkg/daemon/ceph/osd/daemon.go | 11 +- pkg/daemon/ceph/osd/volume.go | 146 +++++++++++++-------- pkg/daemon/ceph/osd/volume_test.go | 85 ++++++++++-- pkg/operator/ceph/cluster/osd/spec.go | 4 +- pkg/operator/ceph/cluster/osd/spec_test.go | 2 +- 6 files changed, 174 insertions(+), 75 deletions(-) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 5ecd46cee..b95ef12fa 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -26,3 +26,4 @@ v1.6... * Extending the support of vault KMS configuration for Ceph RGW * Enable disruption budgets (PDBs) by default for Mon, RGW, MDS, and OSD daemons * Add CephFilesystemMirror CRD to deploy cephfs-mirror daemon +* Ceph OSD: as of Nautilus 14.2.14 and Octopus 15.2.9 if the OSD scenario is simple (one OSD per disk) we won't use LVM to prepare the disk anymore diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 47394f68d..b8415608e 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -196,17 +196,8 @@ func writeCephConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterI // Provision provisions an OSD func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity string) error { - - // Check for the presence of LVM on the host when NOT running on PVC - // since this scenario is still using LVM - if !agent.pvcBacked { - ne := NewNsenter(context, lvmCommandToCheck, []string{"--help"}) - err := ne.checkIfBinaryExistsOnHost() - if err != nil { - return errors.Wrapf(err, "binary %q does not exist on the host, make sure lvm2 package is installed", lvmCommandToCheck) - } + if agent.pvcBacked { // Init KMS store, retrieve the KEK and store it as an env var for ceph-volume - } else { err := setKEKinEnv(context, agent.clusterInfo) if err != nil { return errors.Wrap(err, "failed to set kek as an environment variable") diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index e6b9e21bc..684a6932f 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -27,7 +27,6 @@ import ( "regexp" "strconv" "strings" - "time" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" @@ -57,8 +56,12 @@ var ( cvLogDir = "" // The "ceph-volume raw" command is available since Ceph 14.2.8 as well as partition support in ceph-volume cephVolumeRawModeMinCephVersion = cephver.CephVersion{Major: 14, Minor: 2, Extra: 8} - isEncrypted = os.Getenv(oposd.EncryptedDeviceEnvVarName) == "true" - isOnPVC = os.Getenv(osd.PVCBackedOSDVarName) == "true" + // The Ceph Nautilus to include a retry to acquire device lock + cephFlockFixNautilusMinCephVersion = cephver.CephVersion{Major: 14, Minor: 2, Extra: 14} + // The Ceph Octopus to include a retry to acquire device lock + cephFlockFixOctopusMinCephVersion = cephver.CephVersion{Major: 15, Minor: 2, Extra: 9} + isEncrypted = os.Getenv(oposd.EncryptedDeviceEnvVarName) == "true" + isOnPVC = os.Getenv(osd.PVCBackedOSDVarName) == "true" ) type osdInfoBlock struct { @@ -179,6 +182,13 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device } osds = append(osds, lvmOsds...) + // List existing OSD(s) configured with ceph-volume raw mode + rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, "", "", false) + if err != nil { + logger.Infof("failed to get device already provisioned by ceph-volume raw. %v", err) + } + osds = append(osds, rawOsds...) + return osds, nil } @@ -200,11 +210,17 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device } } - // Update LVM configuration file - // Only do this after Ceph Nautilus 14.2.6 since it will use the ceph-volume raw mode by default and not LVM anymore - if !a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) { - if err := UpdateLVMConfig(context, a.pvcBacked, lvBackedPV); err != nil { - return nil, errors.Wrap(err, "failed to update lvm configuration file") + // Should we use ceph-volume raw mode? + useRawMode, err := a.useRawMode(context, a.pvcBacked) + if err != nil { + return nil, errors.Wrap(err, "failed to determine which ceph-volume mode to use") + } + + // If not raw mode we must execute a few LVM prerequisites + if !useRawMode { + err = lvmPreReq(context, a.pvcBacked, lvBackedPV) + if err != nil { + return nil, errors.Wrap(err, "failed to run lvm prerequisites") } } @@ -214,31 +230,45 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device return nil, errors.Wrap(err, "failed to initialize devices on PVC") } } else { - if err = a.initializeDevices(context, devices); err != nil { - return nil, errors.Wrap(err, "failed to initialize devices") + // Initialize block device OSD without LVM + if useRawMode { + logger.Info("initializing osd disk with raw mode") + err := a.initializeDevicesRawMode(context, devices) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize raw based osd") + } + } else { + // Initialize block device OSD with LVM + logger.Info("initializing osd disk with lvm mode") + err := a.initializeDevicesLVMMode(context, devices) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize lvm based osd") + } } } // List OSD configured with ceph-volume lvm mode + // List THE configured OSD with ceph-volume raw mode + // When the block is encrypted we need to list against the encrypted device mapper + if !isEncrypted { + block = fmt.Sprintf("/mnt/%s", a.nodeName) + } + // List ALL OSDs when not running on PVC + if !a.pvcBacked { + block = "" + } + rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, walBlock, lvBackedPV) + if err != nil { + return nil, errors.Wrap(err, "failed to get devices already provisioned by ceph-volume raw") + } + osds = append(osds, rawOsds...) + lvmOsds, err = GetCephVolumeLVMOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, false, lvBackedPV) if err != nil { return nil, errors.Wrap(err, "failed to get devices already provisioned by ceph-volume lvm") } osds = append(osds, lvmOsds...) - // List THE configured OSD with ceph-volume raw mode - if a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) { - // When the block is encrypted we need to list against the encrypted device mapper - if !isEncrypted { - block = fmt.Sprintf("/mnt/%s", a.nodeName) - } - rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, walBlock, lvBackedPV) - if err != nil { - return nil, errors.Wrap(err, "failed to get devices already provisioned by ceph-volume raw") - } - osds = append(osds, rawOsds...) - } - return osds, err } @@ -461,13 +491,27 @@ func UpdateLVMConfig(context *clusterd.Context, onPVC, lvBackedPV bool) error { return nil } -func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceOsdMapping) error { - // We need at least Ceph 14.2.8 to use ceph-volume raw mode - useRawMode := a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) +func (a *OsdAgent) useRawMode(context *clusterd.Context, pvcBacked bool) (bool, error) { + if pvcBacked { + return a.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion), nil + } + + var useRawMode bool + // Can we safely use ceph-volume raw mode in the non-PVC case? + // On non-PVC we see a race between systemd-udev and the osd process to acquire the lock on the device + if a.clusterInfo.CephVersion.IsNautilus() && a.clusterInfo.CephVersion.IsAtLeast(cephFlockFixNautilusMinCephVersion) { + logger.Debugf("will use raw mode since cluster version is at least %v", cephFlockFixNautilusMinCephVersion) + useRawMode = true + } + + if a.clusterInfo.CephVersion.IsOctopus() && a.clusterInfo.CephVersion.IsAtLeast(cephFlockFixOctopusMinCephVersion) { + logger.Debugf("will use raw mode since cluster version is at least %v", cephFlockFixOctopusMinCephVersion) + useRawMode = true + } // ceph-volume raw mode does not support encryption yet if a.storeConfig.EncryptedDevice { - logger.Infof("won't use raw mode since encryption is enabled") + logger.Debug("won't use raw mode since encryption is enabled") useRawMode = false } @@ -475,42 +519,23 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO osdsPerDeviceCountString := sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice) osdsPerDeviceCount, err := strconv.Atoi(osdsPerDeviceCountString) if err != nil { - return errors.Wrapf(err, "failed to convert string %q to integer", osdsPerDeviceCountString) + return false, errors.Wrapf(err, "failed to convert string %q to integer", osdsPerDeviceCountString) } if osdsPerDeviceCount > 1 { - logger.Infof("won't use raw mode since osd per device > 1 is enabled %d", osdsPerDeviceCount) + logger.Debugf("won't use raw mode since osd per device is %d", osdsPerDeviceCount) useRawMode = false } // ceph-volume raw mode mode does not support metadata device if not running on PVC because the user has specified a whole device if a.metadataDevice != "" { - logger.Infof("won't use raw mode since there is a metadata device %q", a.metadataDevice) + logger.Debugf("won't use raw mode since there is a metadata device %q", a.metadataDevice) useRawMode = false } - // Initialize block device OSD without LVM - if useRawMode { - logger.Info("Initializing OSD disk with raw mode") - err := a.initializeDevicesRawMode(context, devices) - if err != nil { - return errors.Wrap(err, "failed to initialize raw based osd") - } - } else { - // Initialize block device OSD with LVM - logger.Info("Initializing OSD disk with LVM mode") - err := a.initializeDevicesLVMMode(context, devices) - if err != nil { - return errors.Wrap(err, "failed to initialize lvm based osd") - } - } - - return nil + return useRawMode, nil } func (a *OsdAgent) initializeDevicesRawMode(context *clusterd.Context, devices *DeviceOsdMapping) error { - - time.Sleep(time.Hour * 350) - baseCommand := "stdbuf" cephVolumeMode := "raw" baseArgs := []string{"-oL", cephVolumeCmd, cephVolumeMode, "prepare", "--bluestore"} @@ -546,7 +571,6 @@ func (a *OsdAgent) initializeDevicesRawMode(context *clusterd.Context, devices * } return nil - } func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *DeviceOsdMapping) error { @@ -768,6 +792,24 @@ func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices * return nil } +func lvmPreReq(context *clusterd.Context, pvcBacked, lvBackedPV bool) error { + // Check for the presence of LVM on the host when NOT running on PVC + // since this scenario is still using LVM + ne := NewNsenter(context, lvmCommandToCheck, []string{"--help"}) + err := ne.checkIfBinaryExistsOnHost() + if err != nil { + return errors.Wrapf(err, "binary %q does not exist on the host, make sure lvm2 package is installed", lvmCommandToCheck) + } + + // Update LVM configuration file + // Only do this after Ceph Nautilus 14.2.6 since it will use the ceph-volume raw mode by default and not LVM anymore + if err := UpdateLVMConfig(context, pvcBacked, lvBackedPV); err != nil { + return errors.Wrap(err, "failed to update lvm configuration file") + } + + return nil +} + func getDatabaseSize(globalSize int, deviceSize int) int { if deviceSize > 0 { globalSize = deviceSize @@ -890,8 +932,6 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster if block == "" { setDevicePathFromList = true args = []string{cvMode, "list", "--format", "json"} - } else { - blockPath = block } result, err := callCephVolume(context, false, args...) diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 8a04f3f65..55493e8ad 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -28,6 +28,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" + "github.com/rook/rook/pkg/operator/ceph/version" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -339,10 +340,10 @@ func TestConfigureCVDevices(t *testing.T) { if command == "lsblk" && args[0] == mountedDev { return fmt.Sprintf(`SIZE="17179869184" ROTA="1" RO="0" TYPE="lvm" PKNAME="" NAME="%s" KNAME="/dev/dm-1, a ...interface{})`, mapperDev), nil } - if args[1] == "ceph-volume" && args[4] == "lvm" && args[5] == "list" && args[6] == mapperDev { + if contains(args, "lvm") && contains(args, "list") { return `{}`, nil } - if args[1] == "ceph-volume" && args[4] == "raw" && args[5] == "list" && args[6] == mountedDev { + if contains(args, "raw") && contains(args, "list") { return fmt.Sprintf(`{ "0": { "ceph_fsid": "%s", @@ -361,6 +362,9 @@ func TestConfigureCVDevices(t *testing.T) { if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[6] == mapperDev { return initializeBlockPVCTestResult, nil } + if contains(args, "lvm") && contains(args, "list") { + return `{}`, nil + } return "", errors.Errorf("unknown command %s %s", command, args) } @@ -397,6 +401,9 @@ func TestConfigureCVDevices(t *testing.T) { return "", errors.Errorf("unknown command %s %s", command, args) } executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { + if command == "nsenter" { + return "", nil + } logger.Infof("[MockExecuteCommandWithCombinedOutput] %s %v", command, args) return "", errors.Errorf("unknown command %s %s", command, args) } @@ -579,7 +586,7 @@ func TestInitializeBlock(t *testing.T) { a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1"} context := &clusterd.Context{Executor: executor} - err := a.initializeDevices(context, devices) + err := a.initializeDevicesLVMMode(context, devices) assert.NoError(t, err, "failed default behavior test") logger.Info("success, go to next test") } @@ -611,7 +618,7 @@ func TestInitializeBlock(t *testing.T) { a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1", storeConfig: config.StoreConfig{EncryptedDevice: true}} context := &clusterd.Context{Executor: executor} - err := a.initializeDevices(context, devices) + err := a.initializeDevicesLVMMode(context, devices) assert.NoError(t, err, "failed encryption test") logger.Info("success, go to next test") } @@ -643,7 +650,7 @@ func TestInitializeBlock(t *testing.T) { a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1", storeConfig: config.StoreConfig{OSDsPerDevice: 3}} context := &clusterd.Context{Executor: executor} - err := a.initializeDevices(context, devices) + err := a.initializeDevicesLVMMode(context, devices) assert.NoError(t, err, "failed multiple osd test") logger.Info("success, go to next test") } @@ -674,7 +681,7 @@ func TestInitializeBlock(t *testing.T) { } a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1", storeConfig: config.StoreConfig{DeviceClass: "hybrid"}} context := &clusterd.Context{Executor: executor} - err := a.initializeDevices(context, devices) + err := a.initializeDevicesLVMMode(context, devices) assert.NoError(t, err, "failed crush device class test") logger.Info("success, go to next test") } @@ -729,7 +736,7 @@ func TestInitializeBlock(t *testing.T) { a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1"} context := &clusterd.Context{Executor: executor} - err := a.initializeDevices(context, devices) + err := a.initializeDevicesLVMMode(context, devices) assert.NoError(t, err, "failed metadata test") logger.Info("success, go to next test") } @@ -784,7 +791,7 @@ func TestInitializeBlock(t *testing.T) { a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}}, nodeName: "node1"} context := &clusterd.Context{Executor: executor} - err := a.initializeDevices(context, devices) + err := a.initializeDevicesLVMMode(context, devices) assert.NoError(t, err, "failed metadata device by-id test") logger.Info("success, go to next test") } @@ -1241,7 +1248,67 @@ func TestInitializeBlockWithMD(t *testing.T) { a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 15}}, nodeName: "node1"} context := &clusterd.Context{Executor: executor} - err := a.initializeDevices(context, devices) + err := a.initializeDevicesLVMMode(context, devices) assert.NoError(t, err, "failed default behavior test") } } + +func TestUseRawMode(t *testing.T) { + type fields struct { + clusterInfo *cephclient.ClusterInfo + metadataDevice string + storeConfig config.StoreConfig + pvcBacked bool + } + type args struct { + context *clusterd.Context + pvcBacked bool + } + tests := []struct { + name string + fields fields + args args + want bool + wantErr bool + }{ + {"on pvc with lvm", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 5}}, "", config.StoreConfig{}, true}, args{&clusterd.Context{}, true}, false, false}, + {"on pvc with raw", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 8}}, "", config.StoreConfig{}, true}, args{&clusterd.Context{}, true}, true, false}, + {"non-pvc with lvm nautilus", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 13}}, "", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, false, false}, + {"non-pvc with lvm octopus", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 8}}, "", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, false, false}, + {"non-pvc with raw nautilus simple scenario supported", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 14}}, "", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, true, false}, + {"non-pvc with raw octopus simple scenario supported", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 9}}, "", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, true, false}, + {"non-pvc with lvm nautilus complex scenario not supported: encrypted", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 14}}, "", config.StoreConfig{EncryptedDevice: true}, false}, args{&clusterd.Context{}, false}, false, false}, + {"non-pvc with lvm octopus complex scenario not supported: encrypted", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 9}}, "", config.StoreConfig{EncryptedDevice: true}, false}, args{&clusterd.Context{}, false}, false, false}, + {"non-pvc with lvm nautilus complex scenario not supported: osd per device > 1", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 14}}, "", config.StoreConfig{OSDsPerDevice: 2}, false}, args{&clusterd.Context{}, false}, false, false}, + {"non-pvc with lvm octopus complex scenario not supported: osd per device > 1", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 9}}, "", config.StoreConfig{OSDsPerDevice: 2}, false}, args{&clusterd.Context{}, false}, false, false}, + {"non-pvc with lvm nautilus complex scenario not supported: metadata dev", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 14, Minor: 2, Extra: 14}}, "/dev/sdb", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, false, false}, + {"non-pvc with lvm octopus complex scenario not supported: metadata dev", fields{&cephclient.ClusterInfo{CephVersion: version.CephVersion{Major: 15, Minor: 2, Extra: 9}}, "/dev/sdb", config.StoreConfig{}, false}, args{&clusterd.Context{}, false}, false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &OsdAgent{ + clusterInfo: tt.fields.clusterInfo, + metadataDevice: tt.fields.metadataDevice, + storeConfig: tt.fields.storeConfig, + pvcBacked: tt.fields.pvcBacked, + } + got, err := a.useRawMode(tt.args.context, tt.args.pvcBacked) + if (err != nil) != tt.wantErr { + t.Errorf("OsdAgent.useRawMode() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("OsdAgent.useRawMode() = %v, want %v", got, tt.want) + } + }) + } +} + +func contains(arr []string, str string) bool { + for _, a := range arr { + if a == str { + return true + } + } + return false +} diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 9097bc81e..6cc7d566f 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -296,8 +296,8 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC dataDir := k8sutil.DataDir // Create volume config for /dev so the pod can access devices on the host - // Only valid when running OSD with LVM mode - if osd.CVMode == "lvm" { + // Only valid when running OSD with LVM and Raw mode + if !osdProps.onPVC() { devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}} volumes = append(volumes, devVolume) devMount := v1.VolumeMount{Name: "devices", MountPath: "/dev"} diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 7c3d81fe8..40ce50153 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -198,7 +198,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { blkInitCont := deployment.Spec.Template.Spec.InitContainers[2] assert.Equal(t, 1, len(blkInitCont.VolumeDevices)) cont = deployment.Spec.Template.Spec.Containers[0] - assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) + assert.Equal(t, 7, len(cont.VolumeMounts), cont.VolumeMounts) // Test OSD on PVC with RAW osd = OSDInfo{