diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 665d414f2..4c0a93600 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -842,7 +842,7 @@ spec: - ReadWriteOnce ``` -### Dedicated metadata device for OSD on PVC +### Dedicated metadata and wal device for OSD on PVC In the simplest case, Ceph OSD BlueStore consumes a single (primary) storage device. BlueStore is the engine used by the OSD to store data. @@ -881,22 +881,164 @@ So just taking the `storage` section this will give something like: spec: resources: requests: - # Find the right size https://docs.ceph.com/docs/mimic/rados/configuration/bluestore-config-ref/#sizing + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing storage: 5Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) storageClassName: io1 volumeMode: Block accessModes: - ReadWriteOnce ``` -> **NOTE**: Note that Rook only supports two naming convention for a given template: +> **NOTE**: Note that Rook only supports three naming convention for a given template: -* "data": represents the main OSD block device, where your data are being stored -* "metadata": represents the metadata device used to store the Ceph Bluestore database for an OSD. -It is recommended to use a faster storage class for the metadata device, with a slower device for the data. +* "data": represents the main OSD block device, where your data is being stored. +* "metadata": represents the metadata (including block.db and block.wal) device used to store the Ceph Bluestore database for an OSD. +* "wal": represents the block.wal device used to store the Ceph Bluestore database for an OSD. If this device is set, "metadata" device will refer specifically to block.db device. +It is recommended to use a faster storage class for the metadata or wal device, with a slower device for the data. Otherwise, having a separate metadata device will not improve the performance. -To determine the size of the metadata block follow the [official Ceph sizing guide](https://docs.ceph.com/docs/mimic/rados/configuration/bluestore-config-ref/#sizing). + +The bluestore partition has the following reference combinations supported by the ceph-volume utility: + +* A single "data" device. +```yaml + storage: + storageClassDeviceSets: + - name: set1 + count: 3 + portable: false + tuneDeviceClass: false + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce +``` + +* A "data" device and a "metadata" device. +```yaml + storage: + storageClassDeviceSets: + - name: set1 + count: 3 + portable: false + tuneDeviceClass: false + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce + - metadata: + name: metadata + spec: + resources: + requests: + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + storage: 5Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + storageClassName: io1 + volumeMode: Block + accessModes: + - ReadWriteOnce +``` + +* A "data" device and a "wal" device. +A WAL device can be used for BlueStore’s internal journal or write-ahead log (block.wal), it is only useful to use a WAL device if the device is faster than the primary device (data device). +There is no separate "metadata" device in this case, the data of main OSD block and block.db located in "data" device. +```yaml + storage: + storageClassDeviceSets: + - name: set1 + count: 3 + portable: false + tuneDeviceClass: false + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce + - metadata: + name: wal + spec: + resources: + requests: + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + storage: 5Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + storageClassName: io1 + volumeMode: Block + accessModes: + - ReadWriteOnce +``` + +* A "data" device, a "metadata" device and a "wal" device. +```yaml + storage: + storageClassDeviceSets: + - name: set1 + count: 3 + portable: false + tuneDeviceClass: false + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce + - metadata: + name: metadata + spec: + resources: + requests: + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + storage: 5Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + storageClassName: io1 + volumeMode: Block + accessModes: + - ReadWriteOnce + - metadata: + name: wal + spec: + resources: + requests: + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + storage: 5Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + storageClassName: io1 + volumeMode: Block + accessModes: + - ReadWriteOnce +``` + +To determine the size of the metadata block follow the [official Ceph sizing guide](https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing). With the present configuration, each OSD will have its main block allocated a 10GB device as well a 5GB device to act as a bluestore database. diff --git a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml index 57f6ac2a8..fcabe9339 100644 --- a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml @@ -125,8 +125,21 @@ spec: # requests: # # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing # storage: 5Gi - # # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) - # storageClassName: gp2 + # # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + # storageClassName: io1 + # volumeMode: Block + # accessModes: + # - ReadWriteOnce + # dedicated block device to store bluestore wal (block.wal) + # - metadata: + # name: wal + # spec: + # resources: + # requests: + # # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + # storage: 5Gi + # # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + # storageClassName: io1 # volumeMode: Block # accessModes: # - ReadWriteOnce diff --git a/pkg/daemon/ceph/cleanup/disk.go b/pkg/daemon/ceph/cleanup/disk.go index 9c641f009..89c6157d7 100644 --- a/pkg/daemon/ceph/cleanup/disk.go +++ b/pkg/daemon/ceph/cleanup/disk.go @@ -67,7 +67,7 @@ func (s *DiskSanitizer) StartSanitizeDisks() { } // Raw based OSDs - osdRawList, err := osd.GetCephVolumeRawOSDs(s.context, s.clusterInfo, s.clusterInfo.FSID, "", "", false) + osdRawList, err := osd.GetCephVolumeRawOSDs(s.context, s.clusterInfo, s.clusterInfo.FSID, "", "", "", false) if err != nil { logger.Errorf("failed to list raw osd(s). %v", err) } else { diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 78bfe9ae8..4dc64ff29 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -37,6 +37,7 @@ import ( const ( pvcDataTypeDevice = "data" pvcMetadataTypeDevice = "metadata" + pvcWalTypeDevice = "wal" lvmCommandToCheck = "lvm" ) @@ -144,6 +145,28 @@ func killCephOSDProcess(context *clusterd.Context, lvPath string) error { return nil } +func configRawDevice(name string, context *clusterd.Context) (*sys.LocalDisk, error) { + rawDevice, err := clusterd.PopulateDeviceInfo(name, context.Executor) + if err != nil { + return nil, errors.Wrapf(err, "failed to get device info for %q", name) + } + + // set the device type: data, block_db(metadata) or wal. + if strings.HasPrefix(name, "/mnt") { + rawDevice, err = clusterd.PopulateDeviceUdevInfo(rawDevice.KernelName, context.Executor, rawDevice) + if err != nil { + logger.Warningf("failed to get udev info for device %q. %v", name, err) + } + rawDevice.Type = pvcDataTypeDevice + } else if strings.HasPrefix(name, "/srv") { + rawDevice.Type = pvcMetadataTypeDevice + } else if strings.HasPrefix(name, "/wal") { + rawDevice.Type = pvcWalTypeDevice + } + + return rawDevice, nil +} + // Provision provisions an OSD func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) error { @@ -191,29 +214,13 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation string) var rawDevices []*sys.LocalDisk if agent.pvcBacked { - rawDevice, err := clusterd.PopulateDeviceInfo(agent.devices[0].Name, context.Executor) - if err != nil { - return errors.Wrapf(err, "failed to get device info for %q", agent.devices[0].Name) - } - - rawDevice, err = clusterd.PopulateDeviceUdevInfo(rawDevice.KernelName, context.Executor, rawDevice) - if err != nil { - logger.Warningf("failed to get udev info for device %q. %v", agent.devices[0].Name, err) - } - - rawDevice.Type = pvcDataTypeDevice - rawDevices = append(rawDevices, rawDevice) - - // We have a metadata device - if len(agent.devices) > 1 { - rawMetadataDevice, err := clusterd.PopulateDeviceInfo(agent.devices[1].Name, context.Executor) + for i := range agent.devices { + rawDevice, err := configRawDevice(agent.devices[i].Name, context) if err != nil { - return errors.Wrapf(err, "failed to get device info for %q", agent.devices[1].Name) + return err } - // set it's a metadata device - rawMetadataDevice.Type = pvcMetadataTypeDevice - rawDevices = append(rawDevices, rawMetadataDevice) + rawDevices = append(rawDevices, rawDevice) } } else { // We still need to use 'lsblk' as the underlaying way to discover devices @@ -356,7 +363,7 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd rejectedReason := "" if agent.pvcBacked { block := fmt.Sprintf("/mnt/%s", agent.nodeName) - rawOsds, err := GetCephVolumeRawOSDs(context, agent.clusterInfo, agent.clusterInfo.FSID, block, agent.metadataDevice, false) + rawOsds, err := GetCephVolumeRawOSDs(context, agent.clusterInfo, agent.clusterInfo.FSID, block, agent.metadataDevice, "", false) if err != nil { isAvailable = false rejectedReason = fmt.Sprintf("failed to detect if there is already an osd. %v", err) @@ -452,6 +459,12 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd logger.Infof("metadata device %q is selected by the device filter/name %q", device.Name, matchedDevice.Name) deviceInfo = &DeviceOsdIDEntry{Config: matchedDevice, PersistentDevicePaths: strings.Fields(device.DevLinks), Metadata: []int{1}} } + + // set that this is not an OSD but a wal device + if device.Type == pvcWalTypeDevice { + logger.Infof("wal device %q is selected by the device filter/name %q", device.Name, matchedDevice.Name) + deviceInfo = &DeviceOsdIDEntry{Config: matchedDevice, PersistentDevicePaths: strings.Fields(device.DevLinks), Metadata: []int{2}} + } } else { logger.Infof("skipping device %q that does not match the device filter/list (%v). %v", device.Name, desiredDevices, err) } @@ -469,6 +482,8 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd available.Entries[pvcDataTypeDevice] = deviceInfo } else if device.Type == pvcMetadataTypeDevice { available.Entries[pvcMetadataTypeDevice] = deviceInfo + } else if device.Type == pvcWalTypeDevice { + available.Entries[pvcWalTypeDevice] = deviceInfo } } else { available.Entries[device.Name] = deviceInfo diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index fbaa69479..5ab19245e 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -94,7 +94,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device var lvmOsds []oposd.OSDInfo var rawOsds []oposd.OSDInfo var lvBackedPV bool - var block, lvPath, metadataBlock string + var block, lvPath, metadataBlock, walBlock string var err error // Idempotency check, if the device list is empty devices have been prepared already @@ -129,9 +129,9 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device // * restarted the operator, again the prepare job was not re-run // // I'm leaving this code with an empty metadata device for now - metadataBlock = "" + metadataBlock, walBlock = "", "" - rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, lvBackedPV) + rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, walBlock, lvBackedPV) if err != nil { logger.Infof("failed to get device already provisioned by ceph-volume raw. %v", err) } @@ -181,7 +181,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device // If running on OSD on PVC if a.pvcBacked { - if block, metadataBlock, err = a.initializeBlockPVC(context, devices, lvBackedPV); err != nil { + if block, metadataBlock, walBlock, err = a.initializeBlockPVC(context, devices, lvBackedPV); err != nil { return nil, errors.Wrap(err, "failed to initialize devices on PVC") } } else { @@ -203,7 +203,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device if !isEncrypted { block = fmt.Sprintf("/mnt/%s", a.nodeName) } - rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, lvBackedPV) + rawOsds, err = GetCephVolumeRawOSDs(context, a.clusterInfo, a.clusterInfo.FSID, block, metadataBlock, walBlock, lvBackedPV) if err != nil { return nil, errors.Wrap(err, "failed to get devices already provisioned by ceph-volume raw") } @@ -213,7 +213,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device return osds, err } -func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *DeviceOsdMapping, lvBackedPV bool) (string, string, error) { +func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *DeviceOsdMapping, lvBackedPV bool) (string, string, string, error) { // we need to return the block if raw mode is used and the lv if lvm mode baseCommand := "stdbuf" var baseArgs []string @@ -235,9 +235,9 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir, cephVolumeMode, "prepare", "--bluestore"} } - var metadataArg []string - var metadataDev bool - var blockPath, metadataBlockPath string + var metadataArg, walArg []string + var metadataDev, walDev bool + var blockPath, metadataBlockPath, walBlockPath string // Problem: map is an unordered collection // therefore the iteration order of a map is not guaranteed to be the same every time you iterate over it. @@ -245,8 +245,8 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device for name, device := range devices.Entries { // If this is the metadata device there is nothing to do // it'll be used in one of the iterations - if name == "metadata" { - logger.Debugf("device %q is a metadata device, skipping this iteration it will be used in the next one", device.Config.Name) + if name == "metadata" || name == "wal" { + logger.Debugf("device %q is a metadata or wal device, skipping this iteration it will be used in the next one", device.Config.Name) // Don't do this device continue } @@ -263,6 +263,15 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device metadataBlockPath = devices.Entries["metadata"].Config.Name } + if _, ok := devices.Entries["wal"]; ok { + walDev = true + walArg = append(walArg, []string{"--block.wal", + devices.Entries["wal"].Config.Name, + }...) + + walBlockPath = devices.Entries["wal"].Config.Name + } + if device.Data == -1 { logger.Infof("configuring new device %q", device.Config.Name) var err error @@ -272,7 +281,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device // pass 'vg/lv' to ceph-volume deviceArg, err = sys.GetLVName(context.Executor, device.Config.Name) if err != nil { - return "", "", errors.Wrapf(err, "failed to get lv name from device path %q", device.Config.Name) + return "", "", "", errors.Wrapf(err, "failed to get lv name from device path %q", device.Config.Name) } } else { deviceArg = device.Config.Name @@ -304,6 +313,11 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device immediateExecuteArgs = append(immediateExecuteArgs, metadataArg...) } + // Add the cli argument for the wal device + if walDev { + immediateExecuteArgs = append(immediateExecuteArgs, walArg...) + } + // execute ceph-volume with the device op, err := context.Executor.ExecuteCommandWithCombinedOutput(baseCommand, immediateExecuteArgs...) if err != nil { @@ -316,7 +330,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device } // Return failure - return "", "", errors.Wrap(err, "failed ceph-volume") // fail return here as validation provided by ceph-volume + return "", "", "", errors.Wrap(err, "failed ceph-volume") // fail return here as validation provided by ceph-volume } logger.Infof("%v", op) // if raw mode is used or PV on LV, let's return the path of the device @@ -325,12 +339,12 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device } else if cephVolumeMode == "raw" && isEncrypted { blockPath = getEncryptedBlockPath(op) if blockPath == "" { - return "", "", errors.New("failed to get encrypted block path from ceph-volume lvm prepare output") + return "", "", "", errors.New("failed to get encrypted block path from ceph-volume lvm prepare output") } } else { blockPath = getLVPath(op) if blockPath == "" { - return "", "", errors.New("failed to get lv path from ceph-volume lvm prepare output") + return "", "", "", errors.New("failed to get lv path from ceph-volume lvm prepare output") } } } else { @@ -338,7 +352,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device } } - return blockPath, metadataBlockPath, nil + return blockPath, metadataBlockPath, walBlockPath, nil } func getLVPath(op string) string { @@ -695,7 +709,7 @@ func readCVLogContent(cvLogFilePath string) string { } // GetCephVolumeRawOSDs list OSD prepared with raw mode -func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, cephfsid, block, metadataBlock string, lvBackedPV bool) ([]oposd.OSDInfo, error) { +func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, cephfsid, block, metadataBlock, walBlock string, lvBackedPV bool) ([]oposd.OSDInfo, error) { // lv can be a block device if raw mode is used cvMode := "raw" @@ -742,6 +756,7 @@ func GetCephVolumeRawOSDs(context *clusterd.Context, clusterInfo *client.Cluster // Hence, let's use the PVC name instead which will always remain consistent BlockPath: block, MetadataPath: metadataBlock, + WalPath: walBlock, SkipLVRelease: true, LVBackedPV: lvBackedPV, CVMode: cvMode, diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 52b69f21b..481e8e27e 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -292,10 +292,11 @@ func TestInitializeBlockPVC(t *testing.T) { }, } - blockPath, metadataBlockPath, err := a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err := a.initializeBlockPVC(context, devices, false) assert.Nil(t, err) assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath) assert.Equal(t, "", metadataBlockPath) + assert.Equal(t, "", walBlockPath) // Test for failure scenario by giving CephVersion{Major: 14, Minor: 2, Extra: 7} // instead of CephVersion{Major: 14, Minor: 2, Extra: 8}. @@ -309,7 +310,7 @@ func TestInitializeBlockPVC(t *testing.T) { }, } - blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false) assert.NotNil(t, err) executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { @@ -332,10 +333,11 @@ func TestInitializeBlockPVC(t *testing.T) { }, } - blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false) assert.Nil(t, err) assert.Equal(t, "/dev/ceph-bceae560-85b1-4a87-9375-6335fb760c8c/osd-block-2ac8edb0-0d2e-4d8f-a6cc-4c972d56079c", blockPath) assert.Equal(t, "", metadataBlockPath) + assert.Equal(t, "", walBlockPath) // Test for failure scenario by giving CephVersion{Major: 14, Minor: 2, Extra: 8} // instead of cephver.CephVersion{Major: 14, Minor: 2, Extra: 7}. @@ -349,7 +351,7 @@ func TestInitializeBlockPVC(t *testing.T) { }, } - blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false) assert.NotNil(t, err) executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { @@ -373,10 +375,11 @@ func TestInitializeBlockPVC(t *testing.T) { }, } - blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false) assert.Nil(t, err) assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath) assert.Equal(t, "", metadataBlockPath) + assert.Equal(t, "", walBlockPath) // Test for condition when Data !=-1 with CephVersion{Major: 14, Minor: 2, Extra: 8} for raw with flag --crush-device-class. devices = &DeviceOsdMapping{ @@ -385,10 +388,11 @@ func TestInitializeBlockPVC(t *testing.T) { }, } - blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false) assert.Nil(t, err) assert.Equal(t, "", blockPath) assert.Equal(t, "", metadataBlockPath) + assert.Equal(t, "", walBlockPath) } func TestInitializeBlockPVCWithMetadata(t *testing.T) { @@ -412,13 +416,15 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) { Entries: map[string]*DeviceOsdIDEntry{ "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, "metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}}, + "wal": {Data: 1, Metadata: []int{2}, Config: DesiredDevice{Name: ""}}, }, } - blockPath, metadataBlockPath, err := a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err := a.initializeBlockPVC(context, devices, false) assert.Nil(t, err) assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath) assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath) + assert.Equal(t, "", walBlockPath) executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { logger.Infof("%s %v", command, args) @@ -438,13 +444,15 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) { Entries: map[string]*DeviceOsdIDEntry{ "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, "metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}}, + "wal": {Data: 1, Metadata: []int{2}, Config: DesiredDevice{Name: ""}}, }, } - blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false) assert.Nil(t, err) assert.Equal(t, "/dev/ceph-bceae560-85b1-4a87-9375-6335fb760c8c/osd-block-2ac8edb0-0d2e-4d8f-a6cc-4c972d56079c", blockPath) assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath) + assert.Equal(t, "", walBlockPath) executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { logger.Infof("%s %v", command, args) @@ -468,13 +476,15 @@ func TestInitializeBlockPVCWithMetadata(t *testing.T) { Entries: map[string]*DeviceOsdIDEntry{ "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, "metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}}, + "wal": {Data: 1, Metadata: []int{2}, Config: DesiredDevice{Name: ""}}, }, } - blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + blockPath, metadataBlockPath, walBlockPath, err = a.initializeBlockPVC(context, devices, false) assert.Nil(t, err) assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath) assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath) + assert.Equal(t, "", walBlockPath) } func TestParseCephVolumeLVMResult(t *testing.T) { @@ -510,7 +520,7 @@ func TestParseCephVolumeRawResult(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{Namespace: "name"} context := &clusterd.Context{Executor: executor, Clientset: test.New(t, 3)} - osds, err := GetCephVolumeRawOSDs(context, clusterInfo, "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "", "", false) + osds, err := GetCephVolumeRawOSDs(context, clusterInfo, "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", "", "", "", false) assert.Nil(t, err) require.NotNil(t, osds) assert.Equal(t, 2, len(osds)) diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index bedc316c4..08b525db9 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -37,6 +37,7 @@ const ( // Hardcoded in ceph-volume do NOT touch CephVolumeEncryptedKeyEnvVarName = "CEPH_VOLUME_DMCRYPT_SECRET" osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE" + osdWalDeviceEnvVarName = "ROOK_WAL_DEVICE" pvcBackedOSDVarName = "ROOK_PVC_BACKED_OSD" blockPathVarName = "ROOK_BLOCK_PATH" cvModeVarName = "ROOK_CV_MODE" @@ -132,6 +133,10 @@ func metadataDeviceEnvVar(metadataDevice string) v1.EnvVar { return v1.EnvVar{Name: osdMetadataDeviceEnvVarName, Value: metadataDevice} } +func walDeviceEnvVar(walDevice string) v1.EnvVar { + return v1.EnvVar{Name: osdWalDeviceEnvVarName, Value: walDevice} +} + func pvcBackedOSDEnvVar(pvcBacked string) v1.EnvVar { return v1.EnvVar{Name: pvcBackedOSDVarName, Value: pvcBacked} } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index cd48b1195..669581d5d 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -71,6 +71,7 @@ const ( portableKey = "portable" cephOsdPodMinimumMemory uint64 = 2048 // minimum amount of memory in MB to run the pod bluestorePVCMetadata = "metadata" + bluestorePVCWal = "wal" bluestorePVCData = "data" ) @@ -104,6 +105,7 @@ type OSDInfo struct { // BlockPath is the logical Volume path for an OSD created by Ceph-volume with format '/dev//' or simply /dev/vdb if block mode is used BlockPath string `json:"lv-path"` MetadataPath string `json:"metadata-path"` + WalPath string `json:"wal-path"` SkipLVRelease bool `json:"skip-lv-release"` Location string `json:"location"` LVBackedPV bool `json:"lv-backed-pv"` @@ -125,6 +127,7 @@ type osdProperties struct { devices []rookv1.Device pvc v1.PersistentVolumeClaimVolumeSource metadataPVC v1.PersistentVolumeClaimVolumeSource + walPVC v1.PersistentVolumeClaimVolumeSource pvcSize string selection rookv1.Selection resources v1.ResourceRequirements @@ -149,6 +152,10 @@ func (osdProps osdProperties) onPVCWithMetadata() bool { return osdProps.metadataPVC.ClaimName != "" } +func (osdProps osdProperties) onPVCWithWal() bool { + return osdProps.walPVC.ClaimName != "" +} + // Start the osd management func (c *Cluster) Start() error { config := c.newProvisionConfig() @@ -224,10 +231,16 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig) { logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName) } + walSource, walOK := volume.PVCSources[bluestorePVCWal] + if walOK { + logger.Infof("OSD will have its wal device on %q", walSource.ClaimName) + } + osdProps := osdProperties{ crushHostname: dataSource.ClaimName, pvc: dataSource, metadataPVC: metadataSource, + walPVC: walSource, resources: volume.Resources, placement: volume.Placement, portable: volume.Portable, @@ -687,10 +700,16 @@ func (c *Cluster) getOSDPropsForPVC(pvcName string) (osdProperties, error) { logger.Infof("OSD will have its main bluestore block on %q", dataSource.ClaimName) } + walSource, walOK := volumeSource.PVCSources[bluestorePVCWal] + if walOK { + logger.Infof("OSD will have its wal device on %q", walSource.ClaimName) + } + osdProps := osdProperties{ crushHostname: dataSource.ClaimName, pvc: dataSource, metadataPVC: metadataSource, + walPVC: walSource, resources: volumeSource.Resources, placement: volumeSource.Placement, portable: volumeSource.Portable, @@ -786,6 +805,9 @@ func (c *Cluster) getOSDInfo(d *apps.Deployment) ([]OSDInfo, error) { if envVar.Name == osdMetadataDeviceEnvVarName { osd.MetadataPath = envVar.Value } + if envVar.Name == osdWalDeviceEnvVarName { + osd.WalPath = envVar.Value + } } // If CVMode is empty, this likely means we upgraded Rook diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 1268729ed..01d0138dd 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -46,6 +46,9 @@ func (c *Cluster) makeJob(osdProps osdProperties, provisionConfig *provisionConf if osdProps.onPVCWithMetadata() { podSpec.Spec.InitContainers = append(podSpec.Spec.InitContainers, c.getPVCMetadataInitContainer("/srv", osdProps)) } + if osdProps.onPVCWithWal() { + podSpec.Spec.InitContainers = append(podSpec.Spec.InitContainers, c.getPVCWalInitContainer("/wal", osdProps)) + } } job := &batch.Job{ @@ -234,6 +237,14 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun StoreConfig: config.NewStoreConfig(), }) } + if osdProps.onPVCWithWal() { + volumeMounts = append(volumeMounts, getPvcWalOSDBridgeMount(osdProps.walPVC.ClaimName)) + configuredDevices = append(configuredDevices, + config.ConfiguredDevice{ + ID: fmt.Sprintf("/wal/%s", osdProps.walPVC.ClaimName), + StoreConfig: config.NewStoreConfig(), + }) + } marshalledDevices, err := json.Marshal(configuredDevices) if err != nil { return v1.Container{}, errors.Wrapf(err, "failed to JSON marshal configured devices for PVC %q", osdProps.crushHostname) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 8e157aaf3..51c4e548f 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -43,6 +43,7 @@ const ( blockEncryptionOpenInitContainer = "encryption-open" blockPVCMapperEncryptionInitContainer = "blkdevmapper-encryption" blockPVCMetadataMapperInitContainer = "blkdevmapper-metadata" + blockPVCWalMapperInitContainer = "blkdevmapper-wal" activatePVCOSDInitContainer = "activate" expandPVCOSDInitContainer = "expand-bluefs" encryptionKeyFileName = "luks_key" @@ -59,6 +60,7 @@ OSD_DATA_DIR=/var/lib/ceph/osd/ceph-"$OSD_ID" CV_MODE=%s DEVICE=%s METADATA_DEVICE="$%s" +WAL_DEVICE="$%s" # active the osd with ceph-volume if [[ "$CV_MODE" == "lvm" ]]; then @@ -88,6 +90,9 @@ else if [ -n "$METADATA_DEVICE" ]; then ARGS+=(--block.db ${METADATA_DEVICE}) fi + if [ -n "$WAL_DEVICE" ]; then + ARGS+=(--block.wal ${WAL_DEVICE}) + fi # ceph-volume raw mode only supports bluestore so we don't need to pass a store flag ceph-volume "$CV_MODE" activate "${ARGS[@]}" fi @@ -323,6 +328,9 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC if osdProps.onPVCWithMetadata() { initContainers = append(initContainers, c.getPVCMetadataInitContainerActivate(osdDataDirPath, osdProps)) } + if osdProps.onPVCWithWal() { + initContainers = append(initContainers, c.getPVCWalInitContainerActivate(osdDataDirPath, osdProps)) + } } initContainers = append(initContainers, c.getActivatePVCInitContainer(osdProps, osdID)) // Expansion is not supported on encrypted device @@ -490,7 +498,7 @@ func (c *Cluster) getActivateOSDInitContainer(osdID string, osdInfo OSDInfo, osd Command: []string{ "/bin/bash", "-c", - fmt.Sprintf(activateOSDCode, osdID, osdInfo.UUID, osdStore, osdInfo.CVMode, osdInfo.BlockPath, osdMetadataDeviceEnvVarName), + fmt.Sprintf(activateOSDCode, osdID, osdInfo.UUID, osdStore, osdInfo.CVMode, osdInfo.BlockPath, osdMetadataDeviceEnvVarName, osdWalDeviceEnvVarName), }, Name: "activate", Image: c.spec.CephVersion.Image, @@ -642,6 +650,53 @@ func (c *Cluster) getPVCMetadataInitContainerActivate(mountPath string, osdProps } } +func (c *Cluster) getPVCWalInitContainer(mountPath string, osdProps osdProperties) v1.Container { + return v1.Container{ + Name: blockPVCWalMapperInitContainer, + Image: c.spec.CephVersion.Image, + Command: []string{ + "cp", + }, + Args: []string{"-a", fmt.Sprintf("/%s", osdProps.walPVC.ClaimName), fmt.Sprintf("/wal/%s", osdProps.walPVC.ClaimName)}, + VolumeDevices: []v1.VolumeDevice{ + { + Name: osdProps.walPVC.ClaimName, + DevicePath: fmt.Sprintf("/%s", osdProps.walPVC.ClaimName), + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/wal", + Name: fmt.Sprintf("%s-bridge", osdProps.walPVC.ClaimName), + }, + }, + SecurityContext: opmon.PodSecurityContext(), + Resources: osdProps.resources, + } +} + +func (c *Cluster) getPVCWalInitContainerActivate(mountPath string, osdProps osdProperties) v1.Container { + return v1.Container{ + Name: blockPVCWalMapperInitContainer, + Image: c.spec.CephVersion.Image, + Command: []string{ + "cp", + }, + Args: []string{"-a", fmt.Sprintf("/%s", osdProps.walPVC.ClaimName), path.Join(mountPath, "block.wal")}, + VolumeDevices: []v1.VolumeDevice{ + { + Name: osdProps.walPVC.ClaimName, + DevicePath: fmt.Sprintf("/%s", osdProps.walPVC.ClaimName), + }, + }, + // We need to call getPvcOSDBridgeMountActivate() so that we can copy the wal block into the "main" empty dir + // This empty dir is passed along every init container + VolumeMounts: []v1.VolumeMount{getPvcOSDBridgeMountActivate(mountPath, osdProps.pvc.ClaimName)}, + SecurityContext: opmon.PodSecurityContext(), + Resources: osdProps.resources, + } +} + func (c *Cluster) getActivatePVCInitContainer(osdProps osdProperties, osdID string) v1.Container { osdDataPath := activateOSDMountPath + osdID osdDataBlockPath := path.Join(osdDataPath, "block") diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 526a2ea44..da0747cde 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -212,15 +212,17 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { CVMode: "raw", } osdProp.metadataPVC = v1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc-metadata"} + osdProp.walPVC = v1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc-wal"} deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 5, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 6, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) - assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[2].Name) - assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[3].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[4].Name) + assert.Equal(t, "blkdevmapper-wal", deployment.Spec.Template.Spec.InitContainers[2].Name) + assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[4].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 6, len(cont.VolumeMounts), cont.VolumeMounts) diff --git a/pkg/operator/ceph/cluster/osd/volumes.go b/pkg/operator/ceph/cluster/osd/volumes.go index 5c91dfdc5..bd02d9d38 100644 --- a/pkg/operator/ceph/cluster/osd/volumes.go +++ b/pkg/operator/ceph/cluster/osd/volumes.go @@ -54,6 +54,13 @@ func getPvcMetadataOSDBridgeMount(claimName string) v1.VolumeMount { } } +func getPvcWalOSDBridgeMount(claimName string) v1.VolumeMount { + return v1.VolumeMount{ + Name: fmt.Sprintf("%s-bridge", claimName), + MountPath: "/wal", + } +} + func getDeviceMapperMount() v1.VolumeMount { return v1.VolumeMount{ MountPath: dmPath, @@ -123,6 +130,31 @@ func getPVCOSDVolumes(osdProps *osdProperties) []v1.Volume { volumes = append(volumes, metadataPVCVolume...) } + // If we have a wal PVC let's add it + if osdProps.onPVCWithWal() { + walPVCVolume := []v1.Volume{ + { + Name: osdProps.walPVC.ClaimName, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &osdProps.walPVC, + }, + }, + { + // We need a bridge mount which is basically a common volume mount between the non privileged init container + // and the privileged provision container or osd daemon container + // The reason for this is mentioned in the comment for getPVCInitContainer() method + Name: fmt.Sprintf("%s-bridge", osdProps.walPVC.ClaimName), + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: "Memory", + }, + }, + }, + } + + volumes = append(volumes, walPVCVolume...) + } + logger.Debugf("volumes are %+v", volumes) return volumes