diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index bbe2cbabd..531d08881 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -262,10 +262,6 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device devices.Entries["metadata"].Config.Name, }...) - crushDeviceClass := os.Getenv(oposd.CrushDeviceClassVarName) - if crushDeviceClass != "" { - metadataArg = append(metadataArg, []string{crushDeviceClassFlag, crushDeviceClass}...) - } metadataBlockPath = devices.Entries["metadata"].Config.Name } @@ -273,6 +269,7 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device logger.Infof("configuring new device %q", device.Config.Name) var err error var deviceArg string + if lvBackedPV { // pass 'vg/lv' to ceph-volume deviceArg, err = sys.GetLVName(context.Executor, device.Config.Name) @@ -288,6 +285,11 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device deviceArg, }...) + crushDeviceClass := os.Getenv(oposd.CrushDeviceClassVarName) + if crushDeviceClass != "" { + immediateExecuteArgs = append(immediateExecuteArgs, []string{crushDeviceClassFlag, crushDeviceClass}...) + } + // Add the cli argument for the metadata device if metadataDev { immediateExecuteArgs = append(immediateExecuteArgs, metadataArg...) diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index aa0617fdc..30f27529b 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -24,11 +24,19 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +var initializeBlockPVCTestResult = ` + Volume group "ceph-bceae560-85b1-4a87-9375-6335fb760c8c" successfully created + Logical volume "osd-block-2ac8edb0-0d2e-4d8f-a6cc-4c972d56079c" created. +` + var cephVolumeLVMTestResult = `{ "0": [ { @@ -260,6 +268,214 @@ var cephVolumeRAWTestResult = `{ } ` +func TestInitializeBlockPVC(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" { + return initializeBlockPVCTestResult, nil + } + + return "", errors.Errorf("unknown command %s %s", command, args) + } + + // Test with CephVersion{Major: 14, Minor: 2, Extra: 8} for argument raw without flag --crush-device-class. + context := &clusterd.Context{Executor: executor} + clusterInfo := &cephconfig.ClusterInfo{ + CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}, + } + a := &OsdAgent{cluster: clusterInfo, nodeName: "node1"} + devices := &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + }, + } + + blockPath, metadataBlockPath, err := a.initializeBlockPVC(context, devices, false) + assert.Nil(t, err) + assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath) + assert.Equal(t, "", metadataBlockPath) + + // Test for failure scenario by giving CephVersion{Major: 14, Minor: 2, Extra: 7} + // instead of CephVersion{Major: 14, Minor: 2, Extra: 8}. + clusterInfo = &cephconfig.ClusterInfo{ + CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 7}, + } + a = &OsdAgent{cluster: clusterInfo, nodeName: "node1"} + devices = &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + }, + } + + blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + assert.NotNil(t, err) + + executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if args[1] == "ceph-volume" && args[2] == "lvm" && args[3] == "prepare" && args[4] == "--bluestore" { + return initializeBlockPVCTestResult, nil + } + + return "", errors.Errorf("unknown command %s %s", command, args) + } + + // Test with CephVersion{Major: 14, Minor: 2, Extra: 7} for argument lvm without flag --crush-device-class. + clusterInfo = &cephconfig.ClusterInfo{ + CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 7}, + } + a = &OsdAgent{cluster: clusterInfo, nodeName: "node1"} + devices = &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + }, + } + + blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + assert.Nil(t, err) + assert.Equal(t, "/dev/ceph-bceae560-85b1-4a87-9375-6335fb760c8c/osd-block-2ac8edb0-0d2e-4d8f-a6cc-4c972d56079c", blockPath) + assert.Equal(t, "", metadataBlockPath) + + // Test for failure scenario by giving CephVersion{Major: 14, Minor: 2, Extra: 8} + // instead of cephver.CephVersion{Major: 14, Minor: 2, Extra: 7}. + clusterInfo = &cephconfig.ClusterInfo{ + CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}, + } + a = &OsdAgent{cluster: clusterInfo, nodeName: "node1"} + devices = &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + }, + } + + blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + assert.NotNil(t, err) + + executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[7] == "--crush-device-class" { + return initializeBlockPVCTestResult, nil + } + + return "", errors.Errorf("unknown command %s %s", command, args) + } + // Test with CephVersion{Major: 14, Minor: 2, Extra: 8} for argument raw with flag --crush-device-class. + os.Setenv(oposd.CrushDeviceClassVarName, "foo") + defer os.Unsetenv(oposd.CrushDeviceClassVarName) + clusterInfo = &cephconfig.ClusterInfo{ + CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}, + } + a = &OsdAgent{cluster: clusterInfo, nodeName: "node1"} + devices = &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + }, + } + + blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + assert.Nil(t, err) + assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath) + assert.Equal(t, "", metadataBlockPath) + + // Test for condition when Data !=-1 with CephVersion{Major: 14, Minor: 2, Extra: 8} for raw with flag --crush-device-class. + devices = &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: 0, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + }, + } + + blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + assert.Nil(t, err) + assert.Equal(t, "", blockPath) + assert.Equal(t, "", metadataBlockPath) +} + +func TestInitializeBlockPVCWithMetadata(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[7] == "--block.db" { + return initializeBlockPVCTestResult, nil + } + return "", errors.Errorf("unknown command %s %s", command, args) + } + + // Test with CephVersion{Major: 14, Minor: 2, Extra: 8} for argument raw with flag --block.db and without --crush-device-class flag. + context := &clusterd.Context{Executor: executor} + clusterInfo := &cephconfig.ClusterInfo{ + CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}, + } + a := &OsdAgent{cluster: clusterInfo, nodeName: "node1"} + + devices := &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + "metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}}, + }, + } + + blockPath, metadataBlockPath, err := a.initializeBlockPVC(context, devices, false) + assert.Nil(t, err) + assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath) + assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath) + + executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if args[1] == "ceph-volume" && args[2] == "lvm" && args[3] == "prepare" && args[4] == "--bluestore" && args[7] == "--block.db" { + return initializeBlockPVCTestResult, nil + } + return "", errors.Errorf("unknown command %s %s", command, args) + } + + // Test with CephVersion{Major: 14, Minor: 2, Extra: 7} for argument lvm with flag --block.db and without --crush-device-class flag. + clusterInfo = &cephconfig.ClusterInfo{ + CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 7}, + } + a = &OsdAgent{cluster: clusterInfo, nodeName: "node1"} + + devices = &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + "metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}}, + }, + } + + blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + assert.Nil(t, err) + assert.Equal(t, "/dev/ceph-bceae560-85b1-4a87-9375-6335fb760c8c/osd-block-2ac8edb0-0d2e-4d8f-a6cc-4c972d56079c", blockPath) + assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath) + + executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[7] == "--crush-device-class" && args[9] == "--block.db" { + return initializeBlockPVCTestResult, nil + } + + return "", errors.Errorf("unknown command %s %s", command, args) + } + + // Test with CephVersion{Major: 14, Minor: 2, Extra: 8} for argument raw with flag --block.db and --crush-device-class flag. + os.Setenv(oposd.CrushDeviceClassVarName, "foo") + defer os.Unsetenv(oposd.CrushDeviceClassVarName) + context = &clusterd.Context{Executor: executor} + clusterInfo = &cephconfig.ClusterInfo{ + CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}, + } + a = &OsdAgent{cluster: clusterInfo, nodeName: "node1"} + + devices = &DeviceOsdMapping{ + Entries: map[string]*DeviceOsdIDEntry{ + "data": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/mnt/set1-data-0-rpf2k"}}, + "metadata": {Data: 0, Metadata: []int{1}, Config: DesiredDevice{Name: "/srv/set1-metadata-0-8c7kr"}}, + }, + } + + blockPath, metadataBlockPath, err = a.initializeBlockPVC(context, devices, false) + assert.Nil(t, err) + assert.Equal(t, "/mnt/set1-data-0-rpf2k", blockPath) + assert.Equal(t, "/srv/set1-metadata-0-8c7kr", metadataBlockPath) +} + func TestParseCephVolumeLVMResult(t *testing.T) { executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {