Merge pull request #5580 from silverhandy/master

ceph: add support for wal device for OSD on PVC

Documentation/ceph-cluster-crd.md

@@ -842,7 +842,7 @@ spec:
             - ReadWriteOnce
 ```
 
-### Dedicated metadata device for OSD on PVC
+### Dedicated metadata and wal device for OSD on PVC
 
 In the simplest case, Ceph OSD BlueStore consumes a single (primary) storage device.
 BlueStore is the engine used by the OSD to store data.
@@ -881,22 +881,164 @@ So just taking the `storage` section this will give something like:
         spec:
           resources:
             requests:
-              # Find the right size https://docs.ceph.com/docs/mimic/rados/configuration/bluestore-config-ref/#sizing
+              # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing
               storage: 5Gi
-          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1)
           storageClassName: io1
           volumeMode: Block
           accessModes:
             - ReadWriteOnce
 ```
 
-> **NOTE**: Note that Rook only supports two naming convention for a given template:
+> **NOTE**: Note that Rook only supports three naming convention for a given template:
 
-* "data": represents the main OSD block device, where your data are being stored
-* "metadata": represents the metadata device used to store the Ceph Bluestore database for an OSD.
-It is recommended to use a faster storage class for the metadata device, with a slower device for the data.
+* "data": represents the main OSD block device, where your data is being stored.
+* "metadata": represents the metadata (including block.db and block.wal) device used to store the Ceph Bluestore database for an OSD.
+* "wal": represents the block.wal device used to store the Ceph Bluestore database for an OSD. If this device is set, "metadata" device will refer specifically to block.db device.
+It is recommended to use a faster storage class for the metadata or wal device, with a slower device for the data.
 Otherwise, having a separate metadata device will not improve the performance.
-To determine the size of the metadata block follow the [official Ceph sizing guide](https://docs.ceph.com/docs/mimic/rados/configuration/bluestore-config-ref/#sizing).
+
+The bluestore partition has the following reference combinations supported by the ceph-volume utility:
+
+* A single "data" device.
+```yaml
+  storage:
+    storageClassDeviceSets:
+    - name: set1
+      count: 3
+      portable: false
+      tuneDeviceClass: false
+      volumeClaimTemplates:
+      - metadata:
+          name: data
+        spec:
+          resources:
+            requests:
+              storage: 10Gi
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
+          storageClassName: gp2
+          volumeMode: Block
+          accessModes:
+            - ReadWriteOnce
+```
+
+* A "data" device and a "metadata" device.
+```yaml
+  storage:
+    storageClassDeviceSets:
+    - name: set1
+      count: 3
+      portable: false
+      tuneDeviceClass: false
+      volumeClaimTemplates:
+      - metadata:
+          name: data
+        spec:
+          resources:
+            requests:
+              storage: 10Gi
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
+          storageClassName: gp2
+          volumeMode: Block
+          accessModes:
+            - ReadWriteOnce
+      - metadata:
+          name: metadata
+        spec:
+          resources:
+            requests:
+              # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing
+              storage: 5Gi
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1)
+          storageClassName: io1
+          volumeMode: Block
+          accessModes:
+            - ReadWriteOnce
+```
+
+* A "data" device and a "wal" device.
+A WAL device can be used for BlueStore’s internal journal or write-ahead log (block.wal), it is only useful to use a WAL device if the device is faster than the primary device (data device).
+There is no separate "metadata" device in this case, the data of main OSD block and block.db located in "data" device.
+```yaml
+  storage:
+    storageClassDeviceSets:
+    - name: set1
+      count: 3
+      portable: false
+      tuneDeviceClass: false
+      volumeClaimTemplates:
+      - metadata:
+          name: data
+        spec:
+          resources:
+            requests:
+              storage: 10Gi
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
+          storageClassName: gp2
+          volumeMode: Block
+          accessModes:
+            - ReadWriteOnce
+      - metadata:
+          name: wal
+        spec:
+          resources:
+            requests:
+              # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing
+              storage: 5Gi
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1)
+          storageClassName: io1
+          volumeMode: Block
+          accessModes:
+            - ReadWriteOnce
+```
+
+* A "data" device, a "metadata" device and a "wal" device.
+```yaml
+  storage:
+    storageClassDeviceSets:
+    - name: set1
+      count: 3
+      portable: false
+      tuneDeviceClass: false
+      volumeClaimTemplates:
+      - metadata:
+          name: data
+        spec:
+          resources:
+            requests:
+              storage: 10Gi
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
+          storageClassName: gp2
+          volumeMode: Block
+          accessModes:
+            - ReadWriteOnce
+      - metadata:
+          name: metadata
+        spec:
+          resources:
+            requests:
+              # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing
+              storage: 5Gi
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1)
+          storageClassName: io1
+          volumeMode: Block
+          accessModes:
+            - ReadWriteOnce
+      - metadata:
+          name: wal
+        spec:
+          resources:
+            requests:
+              # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing
+              storage: 5Gi
+          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1)
+          storageClassName: io1
+          volumeMode: Block
+          accessModes:
+            - ReadWriteOnce
+```
+
+To determine the size of the metadata block follow the [official Ceph sizing guide](https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing).
 
 With the present configuration, each OSD will have its main block allocated a 10GB device as well a 5GB device to act as a bluestore database.
 

cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml

@@ -125,8 +125,21 @@ spec:
       #       requests:
       #         # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing
       #         storage: 5Gi
-      #     # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
-      #     storageClassName: gp2
+      #     # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1)
+      #     storageClassName: io1
+      #     volumeMode: Block
+      #     accessModes:
+      #       - ReadWriteOnce
+      # dedicated block device to store bluestore wal (block.wal)
+      # - metadata:
+      #     name: wal
+      #   spec:
+      #     resources:
+      #       requests:
+      #         # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing
+      #         storage: 5Gi
+      #     # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1)
+      #     storageClassName: io1
       #     volumeMode: Block
       #     accessModes:
       #       - ReadWriteOnce

pkg/daemon/ceph/cleanup/disk.go

@@ -67,7 +67,7 @@ func (s *DiskSanitizer) StartSanitizeDisks() {
 	}
 
 	// Raw based OSDs
-	osdRawList, err := osd.GetCephVolumeRawOSDs(s.context, s.clusterInfo, s.clusterInfo.FSID, "", "", false)
+	osdRawList, err := osd.GetCephVolumeRawOSDs(s.context, s.clusterInfo, s.clusterInfo.FSID, "", "", "", false)
 	if err != nil {
 		logger.Errorf("failed to list raw osd(s). %v", err)
 	} else {

pkg/daemon/ceph/osd/daemon.go

@@ -37,6 +37,7 @@ import (
 const (
 	pvcDataTypeDevice     = "data"
 	pvcMetadataTypeDevice = "metadata"
+	pvcWalTypeDevice      = "wal"
 	lvmCommandToCheck     = "lvm"
 )
 
@@ -144,6 +145,28 @@ func killCephOSDProcess(context *clusterd.Context, lvPath string) error {
 	return nil
 }
 
+func configRawDevice(name string, context *clusterd.Context) (*sys.LocalDisk, error) {
+	rawDevice, err := clusterd.PopulateDeviceInfo(name, context.Executor)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get device info for %q", name)
+	}
+
+	// set the device type: data, block_db(metadata) or wal.
+	if strings.HasPrefix(name, "/mnt") {
+		rawDevice, err = clusterd.PopulateDeviceUdevInfo(rawDevice.KernelName, context.Executor, rawDevice)
+		if err != nil {
+			logger.Warningf("failed to get udev info for device %q. %v", name, err)
+		}
+		rawDevice.Type = pvcDataTypeDevice
+	} else if strings.HasPrefix(name, "/srv") {
+		rawDevice.Type = pvcMetadataTypeDevice
+	} else if strings.HasPrefix(name, "/wal") {
+		rawDevice.Type = pvcWalTypeDevice
+	}
+
+	return rawDevice, nil
+}
+
 // Provision provisions an OSD
 func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) error {
 
@@ -191,29 +214,13 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string)
 
 	var rawDevices []*sys.LocalDisk
 	if agent.pvcBacked {
-		rawDevice, err := clusterd.PopulateDeviceInfo(agent.devices[0].Name, context.Executor)
-		if err != nil {
-			return errors.Wrapf(err, "failed to get device info for %q", agent.devices[0].Name)
-		}
-
-		rawDevice, err = clusterd.PopulateDeviceUdevInfo(rawDevice.KernelName, context.Executor, rawDevice)
-		if err != nil {
-			logger.Warningf("failed to get udev info for device %q. %v", agent.devices[0].Name, err)
-		}
-
-		rawDevice.Type = pvcDataTypeDevice
-		rawDevices = append(rawDevices, rawDevice)
-
-		// We have a metadata device
-		if len(agent.devices) > 1 {
-			rawMetadataDevice, err := clusterd.PopulateDeviceInfo(agent.devices[1].Name, context.Executor)
+		for i := range agent.devices {
+			rawDevice, err := configRawDevice(agent.devices[i].Name, context)
 			if err != nil {
-				return errors.Wrapf(err, "failed to get device info for %q", agent.devices[1].Name)
+				return err
 			}
 
-			// set it's a metadata device
-			rawMetadataDevice.Type = pvcMetadataTypeDevice
-			rawDevices = append(rawDevices, rawMetadataDevice)
+			rawDevices = append(rawDevices, rawDevice)
 		}
 	} else {
 		// We still need to use 'lsblk' as the underlaying way to discover devices
@@ -356,7 +363,7 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd
 		rejectedReason := ""
 		if agent.pvcBacked {
 			block := fmt.Sprintf("/mnt/%s", agent.nodeName)
-			rawOsds, err := GetCephVolumeRawOSDs(context, agent.clusterInfo, agent.clusterInfo.FSID, block, agent.metadataDevice, false)
+			rawOsds, err := GetCephVolumeRawOSDs(context, agent.clusterInfo, agent.clusterInfo.FSID, block, agent.metadataDevice, "", false)
 			if err != nil {
 				isAvailable = false
 				rejectedReason = fmt.Sprintf("failed to detect if there is already an osd. %v", err)
@@ -452,6 +459,12 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd
 					logger.Infof("metadata device %q is selected by the device filter/name %q", device.Name, matchedDevice.Name)
 					deviceInfo = &DeviceOsdIDEntry{Config: matchedDevice, PersistentDevicePaths: strings.Fields(device.DevLinks), Metadata: []int{1}}
 				}
+
+				// set that this is not an OSD but a wal device
+				if device.Type == pvcWalTypeDevice {
+					logger.Infof("wal device %q is selected by the device filter/name %q", device.Name, matchedDevice.Name)
+					deviceInfo = &DeviceOsdIDEntry{Config: matchedDevice, PersistentDevicePaths: strings.Fields(device.DevLinks), Metadata: []int{2}}
+				}
 			} else {
 				logger.Infof("skipping device %q that does not match the device filter/list (%v). %v", device.Name, desiredDevices, err)
 			}
@@ -469,6 +482,8 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd
 					available.Entries[pvcDataTypeDevice] = deviceInfo
 				} else if device.Type == pvcMetadataTypeDevice {
 					available.Entries[pvcMetadataTypeDevice] = deviceInfo
+				} else if device.Type == pvcWalTypeDevice {
+					available.Entries[pvcWalTypeDevice] = deviceInfo
 				}
 			} else {
 				available.Entries[device.Name] = deviceInfo

pkg/daemon/ceph/osd/volume.go

@@ -94,7 +94,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 	var lvmOsds []oposd.OSDInfo
 	var rawOsds []oposd.OSDInfo
 	var lvBackedPV bool
-	var block, lvPath, metadataBlock string
+	var block, lvPath, metadataBlock, walBlock string
 	var err error
 
 	// Idempotency check, if the device list is empty devices have been prepared already
@@ -129,9 +129,9 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 				//    * restarted the operator, again the prepare job was not re-run
 				//
 				// I'm leaving this code with an empty metadata device for now
-				metadataBlock = ""
+				metadataBlock, walBlock = "", ""
 
-				rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, lvBackedPV)
+				rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, walBlock, lvBackedPV)
 				if err != nil {
 					logger.Infof("failed to get device already provisioned by ceph-volume raw. %v", err)
 				}
@@ -181,7 +181,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 
 	// If running on OSD on PVC
 	if a.pvcBacked {
-		if block, metadataBlock, err = a.initializeBlockPVC(context, devices, lvBackedPV); err != nil {
+		if block, metadataBlock, walBlock, err = a.initializeBlockPVC(context, devices, lvBackedPV); err != nil {
 			return nil, errors.Wrap(err, "failed to initialize devices on PVC")
 		}
 	} else {
@@ -203,7 +203,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 		if !isEncrypted {
 			block = fmt.Sprintf("/mnt/%s", a.nodeName)
 		}
-		rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, lvBackedPV)
+		rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, walBlock, lvBackedPV)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to get devices already provisioned by ceph-volume raw")
 		}
@@ -213,7 +213,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 	return osds, err
 }
 
-func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *DeviceOsdMapping, lvBackedPV bool) (string, string, error) {
+func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *DeviceOsdMapping, lvBackedPV bool) (string, string, string, error) {
 	// we need to return the block if raw mode is used and the lv if lvm mode
 	baseCommand := "stdbuf"
 	var baseArgs []string
@@ -235,9 +235,9 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 		baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir, cephVolumeMode, "prepare", "--bluestore"}
 	}
 
-	var metadataArg []string
-	var metadataDev bool
-	var blockPath, metadataBlockPath string
+	var metadataArg, walArg []string
+	var metadataDev, walDev bool
+	var blockPath, metadataBlockPath, walBlockPath string
 
 	// Problem: map is an unordered collection
 	// therefore the iteration order of a map is not guaranteed to be the same every time you iterate over it.
@@ -245,8 +245,8 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 	for name, device := range devices.Entries {
 		// If this is the metadata device there is nothing to do
 		// it'll be used in one of the iterations
-		if name == "metadata" {
-			logger.Debugf("device %q is a metadata device, skipping this iteration it will be used in the next one", device.Config.Name)
+		if name == "metadata" || name == "wal" {
+			logger.Debugf("device %q is a metadata or wal device, skipping this iteration it will be used in the next one", device.Config.Name)
 			// Don't do this device
 			continue
 		}
@@ -263,6 +263,15 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 			metadataBlockPath = devices.Entries["metadata"].Config.Name
 		}
 
+		if _, ok := devices.Entries["wal"]; ok {
+			walDev = true
+			walArg = append(walArg, []string{"--block.wal",
+				devices.Entries["wal"].Config.Name,
+			}...)
+
+			walBlockPath = devices.Entries["wal"].Config.Name
+		}
+
 		if device.Data == -1 {
 			logger.Infof("configuring new device %q", device.Config.Name)
 			var err error
@@ -272,7 +281,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 				// pass 'vg/lv' to ceph-volume
 				deviceArg, err = sys.GetLVName(context.Executor, device.Config.Name)
 				if err != nil {
-					return "", "", errors.Wrapf(err, "failed to get lv name from device path %q", device.Config.Name)
+					return "", "", "", errors.Wrapf(err, "failed to get lv name from device path %q", device.Config.Name)
 				}
 			} else {
 				deviceArg = device.Config.Name
@@ -304,6 +313,11 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 				immediateExecuteArgs = append(immediateExecuteArgs, metadataArg...)
 			}
 
+			// Add the cli argument for the wal device
+			if walDev {
+				immediateExecuteArgs = append(immediateExecuteArgs, walArg...)
+			}
+
 			// execute ceph-volume with the device
 			op, err := context.Executor.ExecuteCommandWithCombinedOutput(baseCommand, immediateExecuteArgs...)
 			if err != nil {
@@ -316,7 +330,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 				}
 
 				// Return failure
-				return "", "", errors.Wrap(err, "failed ceph-volume") // fail return here as validation provided by ceph-volume
+				return "", "", "", errors.Wrap(err, "failed ceph-volume") // fail return here as validation provided by ceph-volume
 			}
 			logger.Infof("%v", op)
 			// if raw mode is used or PV on LV, let's return the path of the device
@@ -325,12 +339,12 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 			} else if cephVolumeMode == "raw" && isEncrypted {
 				blockPath = getEncryptedBlockPath(op)
 				if blockPath == "" {
-					return "", "", errors.New("failed to get encrypted block path from ceph-volume lvm prepare output")
+					return "", "", "", errors.New("failed to get encrypted block path from ceph-volume lvm prepare output")
 				}
 			} else {
 				blockPath = getLVPath(op)
 				if blockPath == "" {
-					return "", "", errors.New("failed to get lv path from ceph-volume lvm prepare output")
+					return "", "", "", errors.New("failed to get lv path from ceph-volume lvm prepare output")
 				}
 			}
 		} else {
@@ -338,7 +352,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 		}
 	}
 
-	return blockPath, metadataBlockPath, nil
+	return blockPath, metadataBlockPath, walBlockPath, nil
 }
 
 func getLVPath(op string) string {
@@ -695,7 +709,7 @@ func readCVLogContent(cvLogFilePath string) string {
 }
 
 // GetCephVolumeRawOSDs list OSD prepared with raw mode
-func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, cephfsid, block, metadataBlock string, lvBackedPV bool) ([]oposd.OSDInfo, error) {
+func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, cephfsid, block, metadataBlock, walBlock string, lvBackedPV bool) ([]oposd.OSDInfo, error) {
 	// lv can be a block device if raw mode is used
 	cvMode := "raw"
 
@@ -742,6 +756,7 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster
 			// Hence, let's use the PVC name instead which will always remain consistent
 			BlockPath:     block,
 			MetadataPath:  metadataBlock,
+			WalPath:       walBlock,
 			SkipLVRelease: true,
 			LVBackedPV:    lvBackedPV,
 			CVMode:        cvMode,

pkg/daemon/ceph/osd/volume_test.go

@@ -292,10 +292,11 @@ func TestInitializeBlockPVC(t *testing.T) {
 		},
 	}
 
-	blockPath, metadataBlockPath, err := a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err := a.initializeBlockPVC(context, devices, false)
 	assert.Nil(t, err)
 	assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath)
 	assert.Equal(t, "", metadataBlockPath)
+	assert.Equal(t, "", walBlockPath)
 
 	// Test for failure scenario by giving CephVersion{Major: 14, Minor: 2, Extra: 7}
 	// instead of CephVersion{Major: 14, Minor: 2, Extra: 8}.
@@ -309,7 +310,7 @@ func TestInitializeBlockPVC(t *testing.T) {
 		},
 	}
 
-	blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false)
 	assert.NotNil(t, err)
 
 	executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
@@ -332,10 +333,11 @@ func TestInitializeBlockPVC(t *testing.T) {
 		},
 	}
 
-	blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false)
 	assert.Nil(t, err)
 	assert.Equal(t, "/dev/ceph-bceae560-85b1-4a87-9375-6335fb760c8c/osd-block-2ac8edb0-0d2e-4d8f-a6cc-4c972d56079c", blockPath)
 	assert.Equal(t, "", metadataBlockPath)
+	assert.Equal(t, "", walBlockPath)
 
 	// Test for failure scenario by giving CephVersion{Major: 14, Minor: 2, Extra: 8}
 	// instead of cephver.CephVersion{Major: 14, Minor: 2, Extra: 7}.
@@ -349,7 +351,7 @@ func TestInitializeBlockPVC(t *testing.T) {
 		},
 	}
 
-	blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false)
 	assert.NotNil(t, err)
 
 	executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
@@ -373,10 +375,11 @@ func TestInitializeBlockPVC(t *testing.T) {
 		},
 	}
 
-	blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false)
 	assert.Nil(t, err)
 	assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath)
 	assert.Equal(t, "", metadataBlockPath)
+	assert.Equal(t, "", walBlockPath)
 
 	// Test for condition when Data !=-1 with CephVersion{Major: 14, Minor: 2, Extra: 8} for raw  with flag --crush-device-class.
 	devices = &DeviceOsdMapping{
@@ -385,10 +388,11 @@ func TestInitializeBlockPVC(t *testing.T) {
 		},
 	}
 
-	blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false)
 	assert.Nil(t, err)
 	assert.Equal(t, "", blockPath)
 	assert.Equal(t, "", metadataBlockPath)
+	assert.Equal(t, "", walBlockPath)
 }
 
 func TestInitializeBlockPVCWithMetadata(t *testing.T) {
@@ -412,13 +416,15 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) {
 		Entries: map[string]*DeviceOsdIDEntry{
 			"data":     {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}},
 			"metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}},
+			"wal":      {Data: 1, Metadata: []int{2}, Config: DesiredDevice{Name: ""}},
 		},
 	}
 
-	blockPath, metadataBlockPath, err := a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err := a.initializeBlockPVC(context, devices, false)
 	assert.Nil(t, err)
 	assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath)
 	assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath)
+	assert.Equal(t, "", walBlockPath)
 
 	executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
 		logger.Infof("%s %v", command, args)
@@ -438,13 +444,15 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) {
 		Entries: map[string]*DeviceOsdIDEntry{
 			"data":     {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}},
 			"metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}},
+			"wal":      {Data: 1, Metadata: []int{2}, Config: DesiredDevice{Name: ""}},
 		},
 	}
 
-	blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false)
 	assert.Nil(t, err)
 	assert.Equal(t, "/dev/ceph-bceae560-85b1-4a87-9375-6335fb760c8c/osd-block-2ac8edb0-0d2e-4d8f-a6cc-4c972d56079c", blockPath)
 	assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath)
+	assert.Equal(t, "", walBlockPath)
 
 	executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
 		logger.Infof("%s %v", command, args)
@@ -468,13 +476,15 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) {
 		Entries: map[string]*DeviceOsdIDEntry{
 			"data":     {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}},
 			"metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}},
+			"wal":      {Data: 1, Metadata: []int{2}, Config: DesiredDevice{Name: ""}},
 		},
 	}
 
-	blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false)
+	blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false)
 	assert.Nil(t, err)
 	assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath)
 	assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath)
+	assert.Equal(t, "", walBlockPath)
 }
 
 func TestParseCephVolumeLVMResult(t *testing.T) {
@@ -510,7 +520,7 @@ func TestParseCephVolumeRawResult(t *testing.T) {
 	clusterInfo := &cephclient.ClusterInfo{Namespace: "name"}
 
 	context := &clusterd.Context{Executor: executor, Clientset: test.New(t, 3)}
-	osds, err := GetCephVolumeRawOSDs(context, clusterInfo, "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "", "", false)
+	osds, err := GetCephVolumeRawOSDs(context, clusterInfo, "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "", "", "", false)
 	assert.Nil(t, err)
 	require.NotNil(t, osds)
 	assert.Equal(t, 2, len(osds))

pkg/operator/ceph/cluster/osd/envs.go

@@ -37,6 +37,7 @@ const (
 	// Hardcoded in ceph-volume do NOT touch
 	CephVolumeEncryptedKeyEnvVarName = "CEPH_VOLUME_DMCRYPT_SECRET"
 	osdMetadataDeviceEnvVarName      = "ROOK_METADATA_DEVICE"
+	osdWalDeviceEnvVarName           = "ROOK_WAL_DEVICE"
 	pvcBackedOSDVarName              = "ROOK_PVC_BACKED_OSD"
 	blockPathVarName                 = "ROOK_BLOCK_PATH"
 	cvModeVarName                    = "ROOK_CV_MODE"
@@ -132,6 +133,10 @@ func metadataDeviceEnvVar(metadataDevice string) v1.EnvVar {
 	return v1.EnvVar{Name: osdMetadataDeviceEnvVarName, Value: metadataDevice}
 }
 
+func walDeviceEnvVar(walDevice string) v1.EnvVar {
+	return v1.EnvVar{Name: osdWalDeviceEnvVarName, Value: walDevice}
+}
+
 func pvcBackedOSDEnvVar(pvcBacked string) v1.EnvVar {
 	return v1.EnvVar{Name: pvcBackedOSDVarName, Value: pvcBacked}
 }

pkg/operator/ceph/cluster/osd/osd.go

@@ -71,6 +71,7 @@ const (
 	portableKey                         = "portable"
 	cephOsdPodMinimumMemory      uint64 = 2048 // minimum amount of memory in MB to run the pod
 	bluestorePVCMetadata                = "metadata"
+	bluestorePVCWal                     = "wal"
 	bluestorePVCData                    = "data"
 )
 
@@ -104,6 +105,7 @@ type OSDInfo struct {
 	// BlockPath is the logical Volume path for an OSD created by Ceph-volume with format '/dev/<Volume Group>/<Logical Volume>' or simply /dev/vdb if block mode is used
 	BlockPath     string `json:"lv-path"`
 	MetadataPath  string `json:"metadata-path"`
+	WalPath       string `json:"wal-path"`
 	SkipLVRelease bool   `json:"skip-lv-release"`
 	Location      string `json:"location"`
 	LVBackedPV    bool   `json:"lv-backed-pv"`
@@ -125,6 +127,7 @@ type osdProperties struct {
 	devices             []rookv1.Device
 	pvc                 v1.PersistentVolumeClaimVolumeSource
 	metadataPVC         v1.PersistentVolumeClaimVolumeSource
+	walPVC              v1.PersistentVolumeClaimVolumeSource
 	pvcSize             string
 	selection           rookv1.Selection
 	resources           v1.ResourceRequirements
@@ -149,6 +152,10 @@ func (osdProps osdProperties) onPVCWithMetadata() bool {
 	return osdProps.metadataPVC.ClaimName != ""
 }
 
+func (osdProps osdProperties) onPVCWithWal() bool {
+	return osdProps.walPVC.ClaimName != ""
+}
+
 // Start the osd management
 func (c *Cluster) Start() error {
 	config := c.newProvisionConfig()
@@ -224,10 +231,16 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) {
 			logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName)
 		}
 
+		walSource, walOK := volume.PVCSources[bluestorePVCWal]
+		if walOK {
+			logger.Infof("OSD will have its wal device on %q", walSource.ClaimName)
+		}
+
 		osdProps := osdProperties{
 			crushHostname:    dataSource.ClaimName,
 			pvc:              dataSource,
 			metadataPVC:      metadataSource,
+			walPVC:           walSource,
 			resources:        volume.Resources,
 			placement:        volume.Placement,
 			portable:         volume.Portable,
@@ -687,10 +700,16 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) {
 				logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName)
 			}
 
+			walSource, walOK := volumeSource.PVCSources[bluestorePVCWal]
+			if walOK {
+				logger.Infof("OSD will have its wal device on %q", walSource.ClaimName)
+			}
+
 			osdProps := osdProperties{
 				crushHostname:       dataSource.ClaimName,
 				pvc:                 dataSource,
 				metadataPVC:         metadataSource,
+				walPVC:              walSource,
 				resources:           volumeSource.Resources,
 				placement:           volumeSource.Placement,
 				portable:            volumeSource.Portable,
@@ -786,6 +805,9 @@ func (c *Cluster) getOSDInfo(d *apps.Deployment) ([]OSDInfo, error) {
 		if envVar.Name == osdMetadataDeviceEnvVarName {
 			osd.MetadataPath = envVar.Value
 		}
+		if envVar.Name == osdWalDeviceEnvVarName {
+			osd.WalPath = envVar.Value
+		}
 	}
 
 	// If CVMode is empty, this likely means we upgraded Rook

pkg/operator/ceph/cluster/osd/provision_spec.go

@@ -46,6 +46,9 @@ func (c *Cluster) makeJob(osdProps osdProperties, provisionConfig *provisionConf
 		if osdProps.onPVCWithMetadata() {
 			podSpec.Spec.InitContainers = append(podSpec.Spec.InitContainers, c.getPVCMetadataInitContainer("/srv", osdProps))
 		}
+		if osdProps.onPVCWithWal() {
+			podSpec.Spec.InitContainers = append(podSpec.Spec.InitContainers, c.getPVCWalInitContainer("/wal", osdProps))
+		}
 	}
 
 	job := &batch.Job{
@@ -234,6 +237,14 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
 					StoreConfig: config.NewStoreConfig(),
 				})
 		}
+		if osdProps.onPVCWithWal() {
+			volumeMounts = append(volumeMounts, getPvcWalOSDBridgeMount(osdProps.walPVC.ClaimName))
+			configuredDevices = append(configuredDevices,
+				config.ConfiguredDevice{
+					ID:          fmt.Sprintf("/wal/%s", osdProps.walPVC.ClaimName),
+					StoreConfig: config.NewStoreConfig(),
+				})
+		}
 		marshalledDevices, err := json.Marshal(configuredDevices)
 		if err != nil {
 			return v1.Container{}, errors.Wrapf(err, "failed to JSON marshal configured devices for PVC %q", osdProps.crushHostname)

pkg/operator/ceph/cluster/osd/spec.go

@@ -43,6 +43,7 @@ const (
 	blockEncryptionOpenInitContainer      = "encryption-open"
 	blockPVCMapperEncryptionInitContainer = "blkdevmapper-encryption"
 	blockPVCMetadataMapperInitContainer   = "blkdevmapper-metadata"
+	blockPVCWalMapperInitContainer        = "blkdevmapper-wal"
 	activatePVCOSDInitContainer           = "activate"
 	expandPVCOSDInitContainer             = "expand-bluefs"
 	encryptionKeyFileName                 = "luks_key"
@@ -59,6 +60,7 @@ OSD_DATA_DIR=/var/lib/ceph/osd/ceph-"$OSD_ID"
 CV_MODE=%s
 DEVICE=%s
 METADATA_DEVICE="$%s"
+WAL_DEVICE="$%s"
 
 # active the osd with ceph-volume
 if [[ "$CV_MODE" == "lvm" ]]; then
@@ -88,6 +90,9 @@ else
 	if [ -n "$METADATA_DEVICE" ]; then
 		ARGS+=(--block.db ${METADATA_DEVICE})
 	fi
+	if [ -n "$WAL_DEVICE" ]; then
+		ARGS+=(--block.wal ${WAL_DEVICE})
+	fi
 	# ceph-volume raw mode only supports bluestore so we don't need to pass a store flag
 	ceph-volume "$CV_MODE" activate "${ARGS[@]}"
 fi
@@ -323,6 +328,9 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 			if osdProps.onPVCWithMetadata() {
 				initContainers = append(initContainers, c.getPVCMetadataInitContainerActivate(osdDataDirPath, osdProps))
 			}
+			if osdProps.onPVCWithWal() {
+				initContainers = append(initContainers, c.getPVCWalInitContainerActivate(osdDataDirPath, osdProps))
+			}
 		}
 		initContainers = append(initContainers, c.getActivatePVCInitContainer(osdProps, osdID))
 		// Expansion is not supported on encrypted device
@@ -490,7 +498,7 @@ func (c *Cluster) getActivateOSDInitContainer(osdID string, osdInfo OSDInfo, osd
 		Command: []string{
 			"/bin/bash",
 			"-c",
-			fmt.Sprintf(activateOSDCode, osdID, osdInfo.UUID, osdStore, osdInfo.CVMode, osdInfo.BlockPath, osdMetadataDeviceEnvVarName),
+			fmt.Sprintf(activateOSDCode, osdID, osdInfo.UUID, osdStore, osdInfo.CVMode, osdInfo.BlockPath, osdMetadataDeviceEnvVarName, osdWalDeviceEnvVarName),
 		},
 		Name:            "activate",
 		Image:           c.spec.CephVersion.Image,
@@ -642,6 +650,53 @@ func (c *Cluster) getPVCMetadataInitContainerActivate(mountPath string, osdProps
 	}
 }
 
+func (c *Cluster) getPVCWalInitContainer(mountPath string, osdProps osdProperties) v1.Container {
+	return v1.Container{
+		Name:  blockPVCWalMapperInitContainer,
+		Image: c.spec.CephVersion.Image,
+		Command: []string{
+			"cp",
+		},
+		Args: []string{"-a", fmt.Sprintf("/%s", osdProps.walPVC.ClaimName), fmt.Sprintf("/wal/%s", osdProps.walPVC.ClaimName)},
+		VolumeDevices: []v1.VolumeDevice{
+			{
+				Name:       osdProps.walPVC.ClaimName,
+				DevicePath: fmt.Sprintf("/%s", osdProps.walPVC.ClaimName),
+			},
+		},
+		VolumeMounts: []v1.VolumeMount{
+			{
+				MountPath: "/wal",
+				Name:      fmt.Sprintf("%s-bridge", osdProps.walPVC.ClaimName),
+			},
+		},
+		SecurityContext: opmon.PodSecurityContext(),
+		Resources:       osdProps.resources,
+	}
+}
+
+func (c *Cluster) getPVCWalInitContainerActivate(mountPath string, osdProps osdProperties) v1.Container {
+	return v1.Container{
+		Name:  blockPVCWalMapperInitContainer,
+		Image: c.spec.CephVersion.Image,
+		Command: []string{
+			"cp",
+		},
+		Args: []string{"-a", fmt.Sprintf("/%s", osdProps.walPVC.ClaimName), path.Join(mountPath, "block.wal")},
+		VolumeDevices: []v1.VolumeDevice{
+			{
+				Name:       osdProps.walPVC.ClaimName,
+				DevicePath: fmt.Sprintf("/%s", osdProps.walPVC.ClaimName),
+			},
+		},
+		// We need to call getPvcOSDBridgeMountActivate() so that we can copy the wal block into the "main" empty dir
+		// This empty dir is passed along every init container
+		VolumeMounts:    []v1.VolumeMount{getPvcOSDBridgeMountActivate(mountPath, osdProps.pvc.ClaimName)},
+		SecurityContext: opmon.PodSecurityContext(),
+		Resources:       osdProps.resources,
+	}
+}
+
 func (c *Cluster) getActivatePVCInitContainer(osdProps osdProperties, osdID string) v1.Container {
 	osdDataPath := activateOSDMountPath + osdID
 	osdDataBlockPath := path.Join(osdDataPath, "block")

pkg/operator/ceph/cluster/osd/spec_test.go

@@ -212,15 +212,17 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
 		CVMode: "raw",
 	}
 	osdProp.metadataPVC = v1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc-metadata"}
+	osdProp.walPVC = v1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc-wal"}
 	deployment, err = c.makeDeployment(osdProp, osd, dataPathMap)
 	assert.Nil(t, err)
 	assert.NotNil(t, deployment)
-	assert.Equal(t, 5, len(deployment.Spec.Template.Spec.InitContainers))
+	assert.Equal(t, 6, len(deployment.Spec.Template.Spec.InitContainers))
 	assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name)
 	assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name)
-	assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[2].Name)
-	assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[3].Name)
-	assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[4].Name)
+	assert.Equal(t, "blkdevmapper-wal", deployment.Spec.Template.Spec.InitContainers[2].Name)
+	assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[3].Name)
+	assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[4].Name)
+	assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[5].Name)
 	assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers))
 	cont = deployment.Spec.Template.Spec.Containers[0]
 	assert.Equal(t, 6, len(cont.VolumeMounts), cont.VolumeMounts)

pkg/operator/ceph/cluster/osd/volumes.go

@@ -54,6 +54,13 @@ func getPvcMetadataOSDBridgeMount(claimName string) v1.VolumeMount {
 	}
 }
 
+func getPvcWalOSDBridgeMount(claimName string) v1.VolumeMount {
+	return v1.VolumeMount{
+		Name:      fmt.Sprintf("%s-bridge", claimName),
+		MountPath: "/wal",
+	}
+}
+
 func getDeviceMapperMount() v1.VolumeMount {
 	return v1.VolumeMount{
 		MountPath: dmPath,
@@ -123,6 +130,31 @@ func getPVCOSDVolumes(osdProps *osdProperties) []v1.Volume {
 		volumes = append(volumes, metadataPVCVolume...)
 	}
 
+	// If we have a wal PVC let's add it
+	if osdProps.onPVCWithWal() {
+		walPVCVolume := []v1.Volume{
+			{
+				Name: osdProps.walPVC.ClaimName,
+				VolumeSource: v1.VolumeSource{
+					PersistentVolumeClaim: &osdProps.walPVC,
+				},
+			},
+			{
+				// We need a bridge mount which is basically a common volume mount between the non privileged init container
+				// and the privileged provision container or osd daemon container
+				// The reason for this is mentioned in the comment for getPVCInitContainer() method
+				Name: fmt.Sprintf("%s-bridge", osdProps.walPVC.ClaimName),
+				VolumeSource: v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{
+						Medium: "Memory",
+					},
+				},
+			},
+		}
+
+		volumes = append(volumes, walPVCVolume...)
+	}
+
 	logger.Debugf("volumes are %+v", volumes)
 
 	return volumes