From ec2e6ea888b98f18f432c4c5541c17c6fd32ef87 Mon Sep 17 00:00:00 2001 From: Santosh Pillai Date: Wed, 5 Jun 2019 00:34:54 +0530 Subject: [PATCH] use ceph fsid to filter ceph volume OSDs - Updated getCephVolumeOSDs method to use fsid filter while retriving devices. - This solves "failed to fetch mon config (--no-mon-config to skip)" error when multiple ceph clusters are running on same Node Signed-off-by: Santosh Pillai --- pkg/daemon/ceph/osd/agent.go | 2 +- pkg/daemon/ceph/osd/volume.go | 22 +++- pkg/daemon/ceph/osd/volume_test.go | 194 ++++++++++++++++++++++++++++- 3 files changed, 210 insertions(+), 8 deletions(-) diff --git a/pkg/daemon/ceph/osd/agent.go b/pkg/daemon/ceph/osd/agent.go index 6ba8c61b8..90b1c1b61 100644 --- a/pkg/daemon/ceph/osd/agent.go +++ b/pkg/daemon/ceph/osd/agent.go @@ -160,7 +160,7 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs if devices == nil || len(devices.Entries) == 0 { logger.Infof("no more devices to configure") if cvSupported { - return getCephVolumeOSDs(context, a.cluster.Name) + return getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID) } return osds, nil } diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 264a318b6..2a237d817 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -19,7 +19,6 @@ package osd import ( "encoding/json" "fmt" - "github.com/rook/rook/pkg/util/display" "path" "strconv" "syscall" @@ -27,6 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" + "github.com/rook/rook/pkg/util/display" "github.com/rook/rook/pkg/util/exec" ) @@ -46,7 +46,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device var err error if len(devices.Entries) == 0 { logger.Infof("no new devices to configure. returning devices already configured with ceph-volume.") - osds, err = getCephVolumeOSDs(context, a.cluster.Name) + osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID) if err != nil { logger.Infof("failed to get devices already provisioned by ceph-volume. %+v", err) } @@ -62,7 +62,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device return nil, fmt.Errorf("failed to initialize devices. %+v", err) } - osds, err = getCephVolumeOSDs(context, a.cluster.Name) + osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID) return osds, err } @@ -188,7 +188,8 @@ func getCephVolumeSupported(context *clusterd.Context) (bool, error) { return true, nil } -func getCephVolumeOSDs(context *clusterd.Context, clusterName string) ([]oposd.OSDInfo, error) { +func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid string) ([]oposd.OSDInfo, error) { + result, err := context.Executor.ExecuteCommandWithOutput(false, "", cephVolumeCmd, "lvm", "list", "--format", "json") if err != nil { return nil, fmt.Errorf("failed to retrieve ceph-volume results. %+v", err) @@ -211,11 +212,19 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string) ([]oposd.O var osdFSID string isFilestore := false for _, osd := range osdInfo { + if osd.Tags.ClusterFSID != cephfsid { + logger.Infof("skipping osd%d: %s running on a different ceph cluster: %s", id, osd.Tags.OSDFSID, osd.Tags.ClusterFSID) + continue + } osdFSID = osd.Tags.OSDFSID if osd.Type == "journal" { isFilestore = true } } + if len(osdFSID) == 0 { + logger.Infof("Skipping osd%d as no instances are running on ceph cluster: %s", id, cephfsid) + continue + } logger.Infof("osdInfo has %d elements. %+v", len(osdInfo), osdInfo) configDir := "/var/lib/rook/osd" + name @@ -245,6 +254,7 @@ type osdInfo struct { } type osdTags struct { - OSDFSID string `json:"ceph.osd_fsid"` - Encrypted string `json:"ceph.encrypted"` + OSDFSID string `json:"ceph.osd_fsid"` + Encrypted string `json:"ceph.encrypted"` + ClusterFSID string `json:"ceph.cluster_fsid"` } diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 32915ccbf..8020c9038 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -88,6 +88,156 @@ var cephVolumeTestResult = `{ } ` +var cephVolumeTestResultMultiCluster = `{ + "0": [ + { + "devices": [ + "/dev/sdb" + ], + "lv_name": "osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "lv_path": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "lv_size": "<8.00g", + "lv_tags": "ceph.block_device=/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894,ceph.block_uuid=X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=dbe407e0-c1cb-495e-b30a-02e01de6c8ae,ceph.osd_id=0,ceph.type=block,ceph.vdo=0", + "lv_uuid": "X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7", + "name": "osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "path": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "tags": { + "ceph.block_device": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "ceph.block_uuid": "X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7", + "ceph.cephx_lockbox_secret": "", + "ceph.cluster_fsid": "451267e6-883f-4936-8dff-080d781c67d5", + "ceph.cluster_name": "ceph", + "ceph.crush_device_class": "None", + "ceph.encrypted": "0", + "ceph.osd_fsid": "dbe407e0-c1cb-495e-b30a-02e01de6c8ae", + "ceph.osd_id": "0", + "ceph.type": "block", + "ceph.vdo": "0" + }, + "type": "block", + "vg_name": "ceph-93550251-f76c-4219-a33f-df8805de7b9e" + }, + + { + "devices": [ + "/dev/sdc" + ], + "lv_name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "lv_path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "lv_size": "<8.00g", + "lv_tags": "ceph.block_device=/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0,ceph.block_uuid=tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=265d47ca-3e3c-4ef2-ac83-a44b7fb7feee,ceph.osd_id=1,ceph.type=block,ceph.vdo=0", + "lv_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk", + "name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "tags": { + "ceph.block_device": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "ceph.block_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk", + "ceph.cephx_lockbox_secret": "", + "ceph.cluster_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", + "ceph.cluster_name": "ceph", + "ceph.crush_device_class": "None", + "ceph.encrypted": "0", + "ceph.osd_fsid": "265d47ca-3e3c-4ef2-ac83-a44b7fb7feee", + "ceph.osd_id": "1", + "ceph.type": "block", + "ceph.vdo": "0" + }, + "type": "block", + "vg_name": "ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42" + } + ] +} +` +var cephVolumeTestResultMultiClusterMultiOSD = `{ + "0": [ + { + "devices": [ + "/dev/sdb" + ], + "lv_name": "osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "lv_path": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "lv_size": "<8.00g", + "lv_tags": "ceph.block_device=/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894,ceph.block_uuid=X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=dbe407e0-c1cb-495e-b30a-02e01de6c8ae,ceph.osd_id=0,ceph.type=block,ceph.vdo=0", + "lv_uuid": "X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7", + "name": "osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "path": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "tags": { + "ceph.block_device": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894", + "ceph.block_uuid": "X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7", + "ceph.cephx_lockbox_secret": "", + "ceph.cluster_fsid": "451267e6-883f-4936-8dff-080d781c67d5", + "ceph.cluster_name": "ceph", + "ceph.crush_device_class": "None", + "ceph.encrypted": "0", + "ceph.osd_fsid": "dbe407e0-c1cb-495e-b30a-02e01de6c8ae", + "ceph.osd_id": "0", + "ceph.type": "block", + "ceph.vdo": "0" + }, + "type": "block", + "vg_name": "ceph-93550251-f76c-4219-a33f-df8805de7b9e" + }, + + { + "devices": [ + "/dev/sdc" + ], + "lv_name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "lv_path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "lv_size": "<8.00g", + "lv_tags": "ceph.block_device=/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0,ceph.block_uuid=tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=265d47ca-3e3c-4ef2-ac83-a44b7fb7feee,ceph.osd_id=1,ceph.type=block,ceph.vdo=0", + "lv_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk", + "name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "tags": { + "ceph.block_device": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "ceph.block_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk", + "ceph.cephx_lockbox_secret": "", + "ceph.cluster_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", + "ceph.cluster_name": "ceph", + "ceph.crush_device_class": "None", + "ceph.encrypted": "0", + "ceph.osd_fsid": "265d47ca-3e3c-4ef2-ac83-a44b7fb7feee", + "ceph.osd_id": "1", + "ceph.type": "block", + "ceph.vdo": "0" + }, + "type": "block", + "vg_name": "ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42" + } + ], + "1": [ + { + "devices": [ + "/dev/sdc" + ], + "lv_name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "lv_path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "lv_size": "<8.00g", + "lv_tags": "ceph.block_device=/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0,ceph.block_uuid=tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=265d47ca-3e3c-4ef2-ac83-a44b7fb7feee,ceph.osd_id=1,ceph.type=block,ceph.vdo=0", + "lv_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk", + "name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "tags": { + "ceph.block_device": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0", + "ceph.block_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk", + "ceph.cephx_lockbox_secret": "", + "ceph.cluster_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89", + "ceph.cluster_name": "ceph", + "ceph.crush_device_class": "None", + "ceph.encrypted": "0", + "ceph.osd_fsid": "265d47ca-3e3c-4ef2-ac83-a44b7fb7feee", + "ceph.osd_id": "1", + "ceph.type": "block", + "ceph.vdo": "0" + }, + "type": "block", + "vg_name": "ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42" + } + ] +} +` + func TestParseCephVolumeResult(t *testing.T) { executor := &exectest.MockExecutor{} // set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem @@ -102,12 +252,54 @@ func TestParseCephVolumeResult(t *testing.T) { } context := &clusterd.Context{Executor: executor} - osds, err := getCephVolumeOSDs(context, "rook") + osds, err := getCephVolumeOSDs(context, "rook", "4bfe8b72-5e69-4330-b6c0-4d914db8ab89") assert.Nil(t, err) require.NotNil(t, osds) assert.Equal(t, 2, len(osds)) } +func TestCephVolumeResultMultiClusterSingleOSD(t *testing.T) { + executor := &exectest.MockExecutor{} + // set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem + executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { + logger.Infof("%s %+v", command, args) + + if command == "ceph-volume" { + return cephVolumeTestResultMultiCluster, nil + } + + return "", fmt.Errorf("unknown command %s %+v", command, args) + } + + context := &clusterd.Context{Executor: executor} + osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5") + assert.Nil(t, err) + require.NotNil(t, osds) + assert.Equal(t, 1, len(osds)) + assert.Equal(t, osds[0].UUID, "dbe407e0-c1cb-495e-b30a-02e01de6c8ae") +} + +func TestCephVolumeResultMultiClusterMultiOSD(t *testing.T) { + executor := &exectest.MockExecutor{} + // set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem + executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { + logger.Infof("%s %+v", command, args) + + if command == "ceph-volume" { + return cephVolumeTestResultMultiClusterMultiOSD, nil + } + + return "", fmt.Errorf("unknown command %s %+v", command, args) + } + + context := &clusterd.Context{Executor: executor} + osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5") + assert.Nil(t, err) + require.NotNil(t, osds) + assert.Equal(t, 1, len(osds)) + assert.Equal(t, osds[0].UUID, "dbe407e0-c1cb-495e-b30a-02e01de6c8ae") +} + func TestSanitizeOSDsPerDevice(t *testing.T) { assert.Equal(t, "1", sanitizeOSDsPerDevice(-1)) assert.Equal(t, "1", sanitizeOSDsPerDevice(0))