use ceph fsid to filter ceph volume OSDs

- Updated getCephVolumeOSDs method to use fsid filter while retriving devices.
- This solves "failed to fetch mon config (--no-mon-config to skip)" error when multiple ceph clusters are running on same Node

Signed-off-by: Santosh Pillai <sapillai@redhat.com>
This commit is contained in:
Santosh Pillai committed 2019-06-05 12:05:56 +05:30
1 parent cea5fd8676
commit ec2e6ea888
3 files changed
+210 -8

No files matched your search

+1 -1
View File
@@ -160,7 +160,7 @@ func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOs
if devices == nil || len(devices.Entries) == 0 {
logger.Infof("no more devices to configure")
if cvSupported {
return getCephVolumeOSDs(context, a.cluster.Name)
return getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID)
}
return osds, nil
}
+16 -6
View File
@@ -19,7 +19,6 @@ package osd
import (
"encoding/json"
"fmt"
"github.com/rook/rook/pkg/util/display"
"path"
"strconv"
"syscall"
@@ -27,6 +26,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
"github.com/rook/rook/pkg/util/display"
"github.com/rook/rook/pkg/util/exec"
)
@@ -46,7 +46,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
var err error
if len(devices.Entries) == 0 {
logger.Infof("no new devices to configure. returning devices already configured with ceph-volume.")
osds, err = getCephVolumeOSDs(context, a.cluster.Name)
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID)
if err != nil {
logger.Infof("failed to get devices already provisioned by ceph-volume. %+v", err)
}
@@ -62,7 +62,7 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
return nil, fmt.Errorf("failed to initialize devices. %+v", err)
}
osds, err = getCephVolumeOSDs(context, a.cluster.Name)
osds, err = getCephVolumeOSDs(context, a.cluster.Name, a.cluster.FSID)
return osds, err
}
@@ -188,7 +188,8 @@ func getCephVolumeSupported(context *clusterd.Context) (bool, error) {
return true, nil
}
func getCephVolumeOSDs(context *clusterd.Context, clusterName string) ([]oposd.OSDInfo, error) {
func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid string) ([]oposd.OSDInfo, error) {
result, err := context.Executor.ExecuteCommandWithOutput(false, "", cephVolumeCmd, "lvm", "list", "--format", "json")
if err != nil {
return nil, fmt.Errorf("failed to retrieve ceph-volume results. %+v", err)
@@ -211,11 +212,19 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string) ([]oposd.O
var osdFSID string
isFilestore := false
for _, osd := range osdInfo {
if osd.Tags.ClusterFSID != cephfsid {
logger.Infof("skipping osd%d: %s running on a different ceph cluster: %s", id, osd.Tags.OSDFSID, osd.Tags.ClusterFSID)
continue
}
osdFSID = osd.Tags.OSDFSID
if osd.Type == "journal" {
isFilestore = true
}
}
if len(osdFSID) == 0 {
logger.Infof("Skipping osd%d as no instances are running on ceph cluster: %s", id, cephfsid)
continue
}
logger.Infof("osdInfo has %d elements. %+v", len(osdInfo), osdInfo)
configDir := "/var/lib/rook/osd" + name
@@ -245,6 +254,7 @@ type osdInfo struct {
}
type osdTags struct {
OSDFSID string `json:"ceph.osd_fsid"`
Encrypted string `json:"ceph.encrypted"`
OSDFSID string `json:"ceph.osd_fsid"`
Encrypted string `json:"ceph.encrypted"`
ClusterFSID string `json:"ceph.cluster_fsid"`
}
+193 -1
View File
@@ -88,6 +88,156 @@ var cephVolumeTestResult = `{
}
`
var cephVolumeTestResultMultiCluster = `{
"0": [
{
"devices": [
"/dev/sdb"
],
"lv_name": "osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"lv_path": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"lv_size": "<8.00g",
"lv_tags": "ceph.block_device=/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894,ceph.block_uuid=X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=dbe407e0-c1cb-495e-b30a-02e01de6c8ae,ceph.osd_id=0,ceph.type=block,ceph.vdo=0",
"lv_uuid": "X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7",
"name": "osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"path": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"tags": {
"ceph.block_device": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"ceph.block_uuid": "X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7",
"ceph.cephx_lockbox_secret": "",
"ceph.cluster_fsid": "451267e6-883f-4936-8dff-080d781c67d5",
"ceph.cluster_name": "ceph",
"ceph.crush_device_class": "None",
"ceph.encrypted": "0",
"ceph.osd_fsid": "dbe407e0-c1cb-495e-b30a-02e01de6c8ae",
"ceph.osd_id": "0",
"ceph.type": "block",
"ceph.vdo": "0"
},
"type": "block",
"vg_name": "ceph-93550251-f76c-4219-a33f-df8805de7b9e"
},
{
"devices": [
"/dev/sdc"
],
"lv_name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"lv_path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"lv_size": "<8.00g",
"lv_tags": "ceph.block_device=/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0,ceph.block_uuid=tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=265d47ca-3e3c-4ef2-ac83-a44b7fb7feee,ceph.osd_id=1,ceph.type=block,ceph.vdo=0",
"lv_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk",
"name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"tags": {
"ceph.block_device": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"ceph.block_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk",
"ceph.cephx_lockbox_secret": "",
"ceph.cluster_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89",
"ceph.cluster_name": "ceph",
"ceph.crush_device_class": "None",
"ceph.encrypted": "0",
"ceph.osd_fsid": "265d47ca-3e3c-4ef2-ac83-a44b7fb7feee",
"ceph.osd_id": "1",
"ceph.type": "block",
"ceph.vdo": "0"
},
"type": "block",
"vg_name": "ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42"
}
]
}
`
var cephVolumeTestResultMultiClusterMultiOSD = `{
"0": [
{
"devices": [
"/dev/sdb"
],
"lv_name": "osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"lv_path": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"lv_size": "<8.00g",
"lv_tags": "ceph.block_device=/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894,ceph.block_uuid=X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=dbe407e0-c1cb-495e-b30a-02e01de6c8ae,ceph.osd_id=0,ceph.type=block,ceph.vdo=0",
"lv_uuid": "X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7",
"name": "osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"path": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"tags": {
"ceph.block_device": "/dev/ceph-93550251-f76c-4219-a33f-df8805de7b9e/osd-data-d1cb42c3-60f6-4347-82eb-3188dc3df894",
"ceph.block_uuid": "X39Wps-Qewq-d8LV-kj2p-ZqC3-IFQn-C35sV7",
"ceph.cephx_lockbox_secret": "",
"ceph.cluster_fsid": "451267e6-883f-4936-8dff-080d781c67d5",
"ceph.cluster_name": "ceph",
"ceph.crush_device_class": "None",
"ceph.encrypted": "0",
"ceph.osd_fsid": "dbe407e0-c1cb-495e-b30a-02e01de6c8ae",
"ceph.osd_id": "0",
"ceph.type": "block",
"ceph.vdo": "0"
},
"type": "block",
"vg_name": "ceph-93550251-f76c-4219-a33f-df8805de7b9e"
},
{
"devices": [
"/dev/sdc"
],
"lv_name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"lv_path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"lv_size": "<8.00g",
"lv_tags": "ceph.block_device=/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0,ceph.block_uuid=tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=265d47ca-3e3c-4ef2-ac83-a44b7fb7feee,ceph.osd_id=1,ceph.type=block,ceph.vdo=0",
"lv_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk",
"name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"tags": {
"ceph.block_device": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"ceph.block_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk",
"ceph.cephx_lockbox_secret": "",
"ceph.cluster_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89",
"ceph.cluster_name": "ceph",
"ceph.crush_device_class": "None",
"ceph.encrypted": "0",
"ceph.osd_fsid": "265d47ca-3e3c-4ef2-ac83-a44b7fb7feee",
"ceph.osd_id": "1",
"ceph.type": "block",
"ceph.vdo": "0"
},
"type": "block",
"vg_name": "ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42"
}
],
"1": [
{
"devices": [
"/dev/sdc"
],
"lv_name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"lv_path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"lv_size": "<8.00g",
"lv_tags": "ceph.block_device=/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0,ceph.block_uuid=tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=4bfe8b72-5e69-4330-b6c0-4d914db8ab89,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=265d47ca-3e3c-4ef2-ac83-a44b7fb7feee,ceph.osd_id=1,ceph.type=block,ceph.vdo=0",
"lv_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk",
"name": "osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"path": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"tags": {
"ceph.block_device": "/dev/ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42/osd-data-5100eb6b-3a61-4fc1-80ee-86aec275b8b0",
"ceph.block_uuid": "tpdiTi-9Ozq-SrWi-p6od-LohX-s4U0-n2V0vk",
"ceph.cephx_lockbox_secret": "",
"ceph.cluster_fsid": "4bfe8b72-5e69-4330-b6c0-4d914db8ab89",
"ceph.cluster_name": "ceph",
"ceph.crush_device_class": "None",
"ceph.encrypted": "0",
"ceph.osd_fsid": "265d47ca-3e3c-4ef2-ac83-a44b7fb7feee",
"ceph.osd_id": "1",
"ceph.type": "block",
"ceph.vdo": "0"
},
"type": "block",
"vg_name": "ceph-dfb1ca03-eb4f-4a5f-84b4-f4734aaefd42"
}
]
}
`
func TestParseCephVolumeResult(t *testing.T) {
executor := &exectest.MockExecutor{}
// set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem
@@ -102,12 +252,54 @@ func TestParseCephVolumeResult(t *testing.T) {
}
context := &clusterd.Context{Executor: executor}
osds, err := getCephVolumeOSDs(context, "rook")
osds, err := getCephVolumeOSDs(context, "rook", "4bfe8b72-5e69-4330-b6c0-4d914db8ab89")
assert.Nil(t, err)
require.NotNil(t, osds)
assert.Equal(t, 2, len(osds))
}
func TestCephVolumeResultMultiClusterSingleOSD(t *testing.T) {
executor := &exectest.MockExecutor{}
// set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem
executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) {
logger.Infof("%s %+v", command, args)
if command == "ceph-volume" {
return cephVolumeTestResultMultiCluster, nil
}
return "", fmt.Errorf("unknown command %s %+v", command, args)
}
context := &clusterd.Context{Executor: executor}
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5")
assert.Nil(t, err)
require.NotNil(t, osds)
assert.Equal(t, 1, len(osds))
assert.Equal(t, osds[0].UUID, "dbe407e0-c1cb-495e-b30a-02e01de6c8ae")
}
func TestCephVolumeResultMultiClusterMultiOSD(t *testing.T) {
executor := &exectest.MockExecutor{}
// set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem
executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) {
logger.Infof("%s %+v", command, args)
if command == "ceph-volume" {
return cephVolumeTestResultMultiClusterMultiOSD, nil
}
return "", fmt.Errorf("unknown command %s %+v", command, args)
}
context := &clusterd.Context{Executor: executor}
osds, err := getCephVolumeOSDs(context, "rook", "451267e6-883f-4936-8dff-080d781c67d5")
assert.Nil(t, err)
require.NotNil(t, osds)
assert.Equal(t, 1, len(osds))
assert.Equal(t, osds[0].UUID, "dbe407e0-c1cb-495e-b30a-02e01de6c8ae")
}
func TestSanitizeOSDsPerDevice(t *testing.T) {
assert.Equal(t, "1", sanitizeOSDsPerDevice(-1))
assert.Equal(t, "1", sanitizeOSDsPerDevice(0))