osd: detect existing osds not to be passed to ceph-volume

Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
travisn committed 2018-12-07 10:50:37 -07:00
1 parent 1389cc56db
commit fe4a645844
8 files changed
+114 -52

No files matched your search

+1 -1
View File
@@ -125,7 +125,7 @@ func addOSDConfigFlags(command *cobra.Command) {
command.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osdcfg.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)")
command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)")
command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device")
command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encreypt the OSD with dmcrypt")
command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt")
}
func init() {
+48 -20
View File
@@ -145,22 +145,32 @@ func (a *OsdAgent) removeDirs(context *clusterd.Context, removedDirs map[string]
return nil
}
func (a *OsdAgent) configureAllDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, error) {
func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, error) {
// prepare the OSDs with ceph-volume
osds, configured, err := a.configureDevices(context, devices)
cvSupported, err := getCephVolumeSupported(context)
if err != nil {
return nil, fmt.Errorf("failed to configure devices with ceph-volume. %+v", err)
logger.Errorf("failed to detect if ceph-volume is available. %+v", err)
}
if a.metadataDevice != "" {
// ceph-volume still is work in progress for accepting fast devices for the metadata
logger.Infof("skipping ceph-volume until the fast devices can be specified for the metadata")
cvSupported = false
}
if devices == nil || len(devices.Entries) == 0 || configured {
var osds []oposd.OSDInfo
if devices == nil || len(devices.Entries) == 0 {
logger.Infof("no more devices to configure")
if cvSupported {
return getCephVolumeOSDs(context, a.cluster.Name)
}
return osds, nil
}
// Detect OSDs provisioned already with legacy rook
// If ceph-volume is not supported, go ahead and configure the osds natively with rook
// compute an OSD layout scheme that will optimize performance
scheme, err := a.getPartitionPerfScheme(context, devices)
scheme, cvDevices, err := a.getPartitionPerfScheme(context, devices, cvSupported)
logger.Debugf("partition scheme: %+v, err: %+v", scheme, err)
if err != nil {
return osds, fmt.Errorf("failed to get OSD partition scheme: %+v", err)
@@ -173,18 +183,30 @@ func (a *OsdAgent) configureAllDevices(context *clusterd.Context, devices *Devic
}
// initialize and start all the desired OSDs using the computed scheme
succeeded := 0
nonCVTotal := len(scheme.Entries)
for _, entry := range scheme.Entries {
config := &osdConfig{id: entry.ID, uuid: entry.OsdUUID, configRoot: context.ConfigDir,
partitionScheme: entry, storeConfig: a.storeConfig, kv: a.kv, storeName: config.GetConfigStoreName(a.nodeName)}
osd, err := a.prepareOSD(context, config)
if err != nil {
return osds, fmt.Errorf("failed to config osd %d. %+v", entry.ID, err)
} else {
succeeded++
osds = append(osds, *osd)
}
succeeded++
osds = append(osds, *osd)
}
logger.Infof("%d/%d osd devices succeeded on this node", succeeded, len(scheme.Entries))
logger.Infof("%d/%d pre-ceph-volume osd devices succeeded on this node", succeeded, nonCVTotal)
if !cvSupported {
return osds, nil
}
// Now ask ceph-volume for osds already configured or to newly configure devices
cvOSDs, err := a.configureCVDevices(context, cvDevices)
if err != nil {
return nil, fmt.Errorf("failed to configure devices with ceph-volume. %+v", err)
}
osds = append(osds, cvOSDs...)
return osds, nil
}
@@ -224,12 +246,13 @@ func (a *OsdAgent) removeDevices(context *clusterd.Context, removedDevicesScheme
// computes a partitioning scheme for all the given desired devices. This could be devices already in use,
// devices dedicated to metadata, and devices with all bluestore partitions collocated.
func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *DeviceOsdMapping) (*config.PerfScheme, error) {
func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *DeviceOsdMapping, skipNewDevices bool) (*config.PerfScheme, *DeviceOsdMapping, error) {
skippedDevices := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}}
// load the existing (committed) partition scheme from disk
perfScheme, err := config.LoadScheme(a.kv, config.GetConfigStoreName(a.nodeName))
if err != nil {
return nil, fmt.Errorf("failed to load partition scheme: %+v", err)
return nil, skippedDevices, fmt.Errorf("failed to load partition scheme: %+v", err)
}
nameToUUID := map[string]string{}
@@ -254,16 +277,21 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De
logger.Infof("device %s (%s) is already in use", name, nameToUUID)
refreshDeviceInfo(name, nameToUUID, perfScheme)
} else if isDeviceDesiredForData(mapping) {
// device needs data partitioning
logger.Infof("configuring device %s (%s) for data", name, nameToUUID)
numDataNeeded++
if skipNewDevices {
logger.Infof("device %s to be configured by ceph-volume", name)
skippedDevices.Entries[name] = mapping
} else {
// device needs data partitioning
logger.Infof("configuring device %s (%s) for data", name, nameToUUID)
numDataNeeded++
}
} else if isDeviceDesiredForMetadata(mapping, perfScheme) {
// device is desired to store metadata for other OSDs
logger.Infof("configuring device %s (%s) for metadata", name, nameToUUID)
if perfScheme.Metadata != nil {
// TODO: this perf scheme creation algorithm assumes either zero or one metadata device, enhance to allow multiple
// https://github.com/rook/rook/issues/341
return nil, fmt.Errorf("%s is desired for metadata, but %s (%s) is already the metadata device",
return nil, nil, fmt.Errorf("%s is desired for metadata, but %s (%s) is already the metadata device",
name, perfScheme.Metadata.Device, perfScheme.Metadata.DiskUUID)
}
@@ -282,7 +310,7 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De
// register/create the OSD with ceph, which will assign it a cluster wide ID
osdID, osdUUID, err := registerOSD(context, a.cluster.Name)
if err != nil {
return nil, fmt.Errorf("failed to register OSD for device %s: %+v", name, err)
return nil, nil, fmt.Errorf("failed to register OSD for device %s: %+v", name, err)
}
schemeEntry := config.NewPerfSchemeEntry(a.storeConfig.StoreType)
@@ -297,7 +325,7 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De
// populate the perf partition scheme entry with distributed partition details
err := config.PopulateDistributedPerfSchemeEntry(schemeEntry, name, perfScheme.Metadata, a.storeConfig)
if err != nil {
return nil, fmt.Errorf("failed to create distributed perf scheme entry for %s: %+v", name, err)
return nil, nil, fmt.Errorf("failed to create distributed perf scheme entry for %s: %+v", name, err)
}
} else {
// there is no metadata device to use, store everything on the data device
@@ -309,7 +337,7 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De
// populate the perf partition scheme entry with collocated partition details
err := config.PopulateCollocatedPerfSchemeEntry(schemeEntry, name, a.storeConfig)
if err != nil {
return nil, fmt.Errorf("failed to create collocated perf scheme entry for %s: %+v", name, err)
return nil, nil, fmt.Errorf("failed to create collocated perf scheme entry for %s: %+v", name, err)
}
}
@@ -317,7 +345,7 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De
}
}
return perfScheme, nil
return perfScheme, skippedDevices, nil
}
// determines if the given device name is already in use with existing/committed partitions
+42 -7
View File
@@ -120,7 +120,7 @@ func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig config.StoreConfig,
execCount := 0
executor.MockExecuteCommand = func(debug bool, name string, command string, args ...string) error {
logger.Infof("RUN %d: %s %+v", execCount, command, args)
logger.Infof("EXEC %d: %s %+v", execCount, command, args)
parts := strings.Split(name, " ")
nameSuffix := parts[0]
if len(parts) > 1 {
@@ -247,15 +247,22 @@ func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig config.StoreConfig,
"sdx": {Data: -1},
"sdy": {Data: -1},
}}
_, err = agent.configureAllDevices(context, devices)
_, err = agent.configureDevices(context, devices)
assert.Nil(t, err)
assert.Equal(t, int32(0), agent.configCounter)
assert.Equal(t, 0, startCount) // 2 OSD procs should be started
if !legacyProvisioner {
assert.Equal(t, 3, outputExecCount)
assert.Equal(t, 1, execCount)
if storeConfig.StoreType == config.Bluestore {
assert.Equal(t, 7, outputExecCount)
assert.Equal(t, 2, execCount)
} else {
assert.Equal(t, 6, outputExecCount)
// filestore on a device has two more calls than bluestore because of the mount/unmount commands of the legacy sdx device
// where sdy is created as the new c-v osd
assert.Equal(t, 4, execCount)
}
} else if storeConfig.StoreType == config.Bluestore {
assert.Equal(t, 12, outputExecCount) // Bluestore has 2 extra output exec calls to get device properties of each device to determine CRUSH weight
assert.Equal(t, 5, execCount) // 1 osd mkfs for sdx, 3 partition steps for sdy, 1 osd mkfs for sdy
@@ -374,8 +381,19 @@ func createTestAgent(t *testing.T, devices, configDir, nodeName string, storeCon
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
logger.Infof("%s %v", command, args)
return "{\"key\":\"mysecurekey\", \"osdid\":3.0}", nil
},
MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, args ...string) (string, error) {
logger.Infof("%s %v", command, args)
if command == "ceph-volume" {
if len(args) == 3 && args[0] == "lvm" && args[1] == "batch" && args[2] == "--prepare" {
logger.Infof("test c-v not supported")
return "", fmt.Errorf("c-v not supported")
}
}
return "", nil
},
}
cluster := &cephconfig.ClusterInfo{Name: "myclust"}
context := &clusterd.Context{ConfigDir: configDir, Executor: executor, Clientset: testop.New(1)}
@@ -449,7 +467,7 @@ func TestGetPartitionPerfScheme(t *testing.T) {
devices, err := getAvailableDevices(context, []DesiredDevice{{Name: "sda"}, {Name: "sdb"}}, "sdc")
assert.Nil(t, err)
scheme, err := a.getPartitionPerfScheme(context, devices)
scheme, _, err := a.getPartitionPerfScheme(context, devices, false)
assert.Nil(t, err)
assert.Equal(t, 2, len(scheme.Entries))
@@ -525,7 +543,7 @@ func TestGetPartitionSchemeDiskInUse(t *testing.T) {
// get the partition scheme based on the available devices. Since sda is already in use, the partition
// scheme returned should reflect that.
devices, err := getAvailableDevices(context, []DesiredDevice{{Name: "sda"}}, "")
scheme, err := a.getPartitionPerfScheme(context, devices)
scheme, _, err := a.getPartitionPerfScheme(context, devices, false)
assert.Nil(t, err)
// the partition scheme should have a single entry for osd 1 on sda and it should have collocated data and metadata
@@ -593,13 +611,30 @@ func TestGetPartitionSchemeDiskNameChanged(t *testing.T) {
// get the current partition scheme. This should notice that the device names changed and update the
// partition scheme to have the latest device names
devices, err := getAvailableDevices(context, []DesiredDevice{{Name: "sda-changed"}}, "nvme01")
scheme, err := a.getPartitionPerfScheme(context, devices)
scheme, _, err := a.getPartitionPerfScheme(context, devices, false)
assert.Nil(t, err)
require.NotNil(t, scheme)
assert.Equal(t, "sda-changed", scheme.Entries[0].Partitions[config.BlockPartitionType].Device)
assert.Equal(t, "nvme01", scheme.Metadata.Device)
assert.Equal(t, "nvme01", scheme.Entries[0].Partitions[config.WalPartitionType].Device)
assert.Equal(t, "nvme01", scheme.Entries[0].Partitions[config.DatabasePartitionType].Device)
// new devices should be skipped for ceph-volume to configure.
logger.Infof("testing skipping new devices that should be configured by ceph-volume instead of with legacy")
context.Devices = []*sys.LocalDisk{
{Name: "sdx"},
}
devices = &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{"nvme05": {Data: -1, Config: DesiredDevice{Name: "nvme05", OSDsPerDevice: 5}}}}
scheme, skipped, err := a.getPartitionPerfScheme(context, devices, true)
assert.Nil(t, err)
require.NotNil(t, scheme)
require.Equal(t, 1, len(skipped.Entries))
require.Equal(t, 1, len(scheme.Entries))
assert.Equal(t, "nvme05", skipped.Entries["nvme05"].Config.Name)
assert.Equal(t, 5, skipped.Entries["nvme05"].Config.OSDsPerDevice)
for _, p := range scheme.Entries[0].Partitions {
assert.NotEqual(t, "nvme05", p.Device)
}
}
func TestPrepareOSDRoot(t *testing.T) {
+1 -1
View File
@@ -125,7 +125,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent) error {
// start the desired OSDs on devices
logger.Infof("configuring osd devices: %+v", devices)
deviceOSDs, err := agent.configureAllDevices(context, devices)
deviceOSDs, err := agent.configureDevices(context, devices)
if err != nil {
return fmt.Errorf("failed to configure devices. %+v", err)
}
+1 -1
View File
@@ -73,7 +73,7 @@ func TestOSDBootstrap(t *testing.T) {
err := createOSDBootstrapKeyring(context, clusterName, configDir)
assert.Nil(t, err)
targetPath := path.Join(configDir, "bootstrap-osd/ceph.keyring")
targetPath := path.Join(configDir, bootstrapOsdKeyring)
contents, err := ioutil.ReadFile(targetPath)
assert.Nil(t, err)
assert.NotEqual(t, -1, strings.Index(string(contents), "[client.bootstrap-osd]"))
+3 -2
View File
@@ -29,7 +29,8 @@ import (
)
const (
keyringFileName = "keyring"
keyringFileName = "keyring"
bootstrapOsdKeyring = "bootstrap-osd/ceph.keyring"
)
// get the bootstrap OSD root dir
@@ -64,7 +65,7 @@ func getOSDTempMonMapPath(osdDataPath string) string {
// create a keyring for the bootstrap-osd client, it gets a limited set of privileges
func createOSDBootstrapKeyring(context *clusterd.Context, clusterName, rootDir string) error {
username := "client.bootstrap-osd"
keyringPath := path.Join(rootDir, "bootstrap-osd/ceph.keyring")
keyringPath := path.Join(rootDir, bootstrapOsdKeyring)
access := []string{"mon", "allow profile bootstrap-osd"}
keyringEval := func(key string) string {
return fmt.Sprintf(bootstrapOSDKeyringTemplate, key)
+15 -19
View File
@@ -37,37 +37,30 @@ const (
cephVolumeCmd = "ceph-volume"
)
func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, bool, error) {
func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, error) {
var osds []oposd.OSDInfo
useCephVolume, err := getCephVolumeSupported(context)
if err != nil {
return nil, false, fmt.Errorf("failed to detect if ceph-volume is available. %+v", err)
}
if !useCephVolume {
return osds, false, nil
}
if devices == nil || len(devices.Entries) == 0 {
var err error
if len(devices.Entries) == 0 {
logger.Infof("no new devices to configure. returning devices already configured with ceph-volume.")
osds, err = getCephVolumeOSDs(context, a.cluster.Name)
if err != nil {
logger.Infof("failed to get devices already provisioned by ceph-volume. %+v", err)
}
return osds, true, nil
return osds, nil
}
err = createOSDBootstrapKeyring(context, a.cluster.Name, cephConfigDir)
if err != nil {
return nil, true, fmt.Errorf("failed to generate osd keyring. %+v", err)
return nil, fmt.Errorf("failed to generate osd keyring. %+v", err)
}
if err = a.initializeDevices(context, devices); err != nil {
return nil, true, fmt.Errorf("failed to initialize devices. %+v", err)
return nil, fmt.Errorf("failed to initialize devices. %+v", err)
}
osds, err = getCephVolumeOSDs(context, a.cluster.Name)
return osds, true, err
return osds, err
}
func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceOsdMapping) error {
@@ -86,6 +79,10 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
strconv.Itoa(a.storeConfig.OSDsPerDevice),
}...)
// ceph-volume is soon implementing a parameter to specify the "fast devices", which correspond to the "metadataDevice" from the
// crd spec. After that is implemented, we can implement this. In the meantime, we fall back to use rook's partitioning.
metadataDeviceSpecified := false
configured := 0
for name, device := range devices.Entries {
if device.LegacyPartitionsFound {
@@ -96,7 +93,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
if device.Data == -1 {
logger.Infof("configuring new device %s", name)
deviceArg := path.Join("/dev", name)
if device.Config.OSDsPerDevice <= 1 {
if metadataDeviceSpecified {
// the device will be configured as a batch at the end of the method
batchArgs = append(batchArgs, deviceArg)
configured++
@@ -135,10 +132,9 @@ func getCephVolumeSupported(context *clusterd.Context) (bool, error) {
logger.Infof("supported version of ceph-volume not available")
return false, nil
}
logger.Warningf("unknown return code from ceph-volume when checking for compatibility: %d", exitStatus)
return false, fmt.Errorf("unknown return code from ceph-volume when checking for compatibility: %d", exitStatus)
}
logger.Warningf("unknown ceph-volume failure. %+v", err)
return false, nil
return false, fmt.Errorf("unknown ceph-volume failure. %+v", err)
}
return true, nil
@@ -188,7 +184,7 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string) ([]oposd.O
osds = append(osds, osd)
}
logger.Infof("%d osd devices configured on this node", len(osds))
logger.Infof("%d ceph-volume osd devices configured on this node", len(osds))
return osds, nil
}
+3 -1
View File
@@ -189,7 +189,7 @@ func (c *ClusterController) onAdd(obj interface{}) {
if !cluster.Spec.CephVersion.AllowUnsupported {
if !versionSupported(cluster.Spec.CephVersion.Name) {
logger.Errorf("unsupported ceph version detected: %s. allowUnupported must be set to true to run with this version.", cluster.Spec.CephVersion.Name)
logger.Errorf("unsupported ceph version detected: %s. allowUnsupported must be set to true to run with this version.", cluster.Spec.CephVersion.Name)
return
}
}
@@ -321,8 +321,10 @@ func (c *ClusterController) onUpdate(oldObj, newObj interface{}) {
// if the image changed, we need to detect the new image version
if oldClust.Spec.CephVersion.Image != newClust.Spec.CephVersion.Image {
logger.Infof("the ceph version changed. detecting the new image version...")
cluster.Spec.CephVersion.Image = newClust.Spec.CephVersion.Image
if err := cluster.setCephMajorVersion(15 * time.Minute); err != nil {
logger.Errorf("unknown ceph major version. %+v", err)
cluster.Spec.CephVersion.Image = oldClust.Spec.CephVersion.Image
return
}
} else {