diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index dc494aa58..ef9e01bd7 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -125,7 +125,7 @@ func addOSDConfigFlags(command *cobra.Command) { command.Flags().IntVar(&cfg.storeConfig.JournalSizeMB, "osd-journal-size", osdcfg.JournalDefaultSizeMB, "default size (MB) for OSD journal (filestore)") command.Flags().StringVar(&cfg.storeConfig.StoreType, "osd-store", "", "type of backing OSD store to use (bluestore or filestore)") command.Flags().IntVar(&cfg.storeConfig.OSDsPerDevice, "osds-per-device", 1, "the number of OSDs per device") - command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encreypt the OSD with dmcrypt") + command.Flags().BoolVar(&cfg.storeConfig.EncryptedDevice, "encrypted-device", false, "whether to encrypt the OSD with dmcrypt") } func init() { diff --git a/pkg/daemon/ceph/osd/agent.go b/pkg/daemon/ceph/osd/agent.go index 2d9db734a..62642f76f 100644 --- a/pkg/daemon/ceph/osd/agent.go +++ b/pkg/daemon/ceph/osd/agent.go @@ -145,22 +145,32 @@ func (a *OsdAgent) removeDirs(context *clusterd.Context, removedDirs map[string] return nil } -func (a *OsdAgent) configureAllDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, error) { +func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, error) { - // prepare the OSDs with ceph-volume - osds, configured, err := a.configureDevices(context, devices) + cvSupported, err := getCephVolumeSupported(context) if err != nil { - return nil, fmt.Errorf("failed to configure devices with ceph-volume. %+v", err) + logger.Errorf("failed to detect if ceph-volume is available. %+v", err) + } + if a.metadataDevice != "" { + // ceph-volume still is work in progress for accepting fast devices for the metadata + logger.Infof("skipping ceph-volume until the fast devices can be specified for the metadata") + cvSupported = false } - if devices == nil || len(devices.Entries) == 0 || configured { + var osds []oposd.OSDInfo + if devices == nil || len(devices.Entries) == 0 { + logger.Infof("no more devices to configure") + if cvSupported { + return getCephVolumeOSDs(context, a.cluster.Name) + } return osds, nil } + // Detect OSDs provisioned already with legacy rook // If ceph-volume is not supported, go ahead and configure the osds natively with rook // compute an OSD layout scheme that will optimize performance - scheme, err := a.getPartitionPerfScheme(context, devices) + scheme, cvDevices, err := a.getPartitionPerfScheme(context, devices, cvSupported) logger.Debugf("partition scheme: %+v, err: %+v", scheme, err) if err != nil { return osds, fmt.Errorf("failed to get OSD partition scheme: %+v", err) @@ -173,18 +183,30 @@ func (a *OsdAgent) configureAllDevices(context *clusterd.Context, devices *Devic } // initialize and start all the desired OSDs using the computed scheme succeeded := 0 + nonCVTotal := len(scheme.Entries) for _, entry := range scheme.Entries { config := &osdConfig{id: entry.ID, uuid: entry.OsdUUID, configRoot: context.ConfigDir, partitionScheme: entry, storeConfig: a.storeConfig, kv: a.kv, storeName: config.GetConfigStoreName(a.nodeName)} osd, err := a.prepareOSD(context, config) if err != nil { return osds, fmt.Errorf("failed to config osd %d. %+v", entry.ID, err) - } else { - succeeded++ - osds = append(osds, *osd) } + + succeeded++ + osds = append(osds, *osd) } - logger.Infof("%d/%d osd devices succeeded on this node", succeeded, len(scheme.Entries)) + logger.Infof("%d/%d pre-ceph-volume osd devices succeeded on this node", succeeded, nonCVTotal) + + if !cvSupported { + return osds, nil + } + + // Now ask ceph-volume for osds already configured or to newly configure devices + cvOSDs, err := a.configureCVDevices(context, cvDevices) + if err != nil { + return nil, fmt.Errorf("failed to configure devices with ceph-volume. %+v", err) + } + osds = append(osds, cvOSDs...) return osds, nil } @@ -224,12 +246,13 @@ func (a *OsdAgent) removeDevices(context *clusterd.Context, removedDevicesScheme // computes a partitioning scheme for all the given desired devices. This could be devices already in use, // devices dedicated to metadata, and devices with all bluestore partitions collocated. -func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *DeviceOsdMapping) (*config.PerfScheme, error) { +func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *DeviceOsdMapping, skipNewDevices bool) (*config.PerfScheme, *DeviceOsdMapping, error) { + skippedDevices := &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{}} // load the existing (committed) partition scheme from disk perfScheme, err := config.LoadScheme(a.kv, config.GetConfigStoreName(a.nodeName)) if err != nil { - return nil, fmt.Errorf("failed to load partition scheme: %+v", err) + return nil, skippedDevices, fmt.Errorf("failed to load partition scheme: %+v", err) } nameToUUID := map[string]string{} @@ -254,16 +277,21 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De logger.Infof("device %s (%s) is already in use", name, nameToUUID) refreshDeviceInfo(name, nameToUUID, perfScheme) } else if isDeviceDesiredForData(mapping) { - // device needs data partitioning - logger.Infof("configuring device %s (%s) for data", name, nameToUUID) - numDataNeeded++ + if skipNewDevices { + logger.Infof("device %s to be configured by ceph-volume", name) + skippedDevices.Entries[name] = mapping + } else { + // device needs data partitioning + logger.Infof("configuring device %s (%s) for data", name, nameToUUID) + numDataNeeded++ + } } else if isDeviceDesiredForMetadata(mapping, perfScheme) { // device is desired to store metadata for other OSDs logger.Infof("configuring device %s (%s) for metadata", name, nameToUUID) if perfScheme.Metadata != nil { // TODO: this perf scheme creation algorithm assumes either zero or one metadata device, enhance to allow multiple // https://github.com/rook/rook/issues/341 - return nil, fmt.Errorf("%s is desired for metadata, but %s (%s) is already the metadata device", + return nil, nil, fmt.Errorf("%s is desired for metadata, but %s (%s) is already the metadata device", name, perfScheme.Metadata.Device, perfScheme.Metadata.DiskUUID) } @@ -282,7 +310,7 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De // register/create the OSD with ceph, which will assign it a cluster wide ID osdID, osdUUID, err := registerOSD(context, a.cluster.Name) if err != nil { - return nil, fmt.Errorf("failed to register OSD for device %s: %+v", name, err) + return nil, nil, fmt.Errorf("failed to register OSD for device %s: %+v", name, err) } schemeEntry := config.NewPerfSchemeEntry(a.storeConfig.StoreType) @@ -297,7 +325,7 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De // populate the perf partition scheme entry with distributed partition details err := config.PopulateDistributedPerfSchemeEntry(schemeEntry, name, perfScheme.Metadata, a.storeConfig) if err != nil { - return nil, fmt.Errorf("failed to create distributed perf scheme entry for %s: %+v", name, err) + return nil, nil, fmt.Errorf("failed to create distributed perf scheme entry for %s: %+v", name, err) } } else { // there is no metadata device to use, store everything on the data device @@ -309,7 +337,7 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De // populate the perf partition scheme entry with collocated partition details err := config.PopulateCollocatedPerfSchemeEntry(schemeEntry, name, a.storeConfig) if err != nil { - return nil, fmt.Errorf("failed to create collocated perf scheme entry for %s: %+v", name, err) + return nil, nil, fmt.Errorf("failed to create collocated perf scheme entry for %s: %+v", name, err) } } @@ -317,7 +345,7 @@ func (a *OsdAgent) getPartitionPerfScheme(context *clusterd.Context, devices *De } } - return perfScheme, nil + return perfScheme, skippedDevices, nil } // determines if the given device name is already in use with existing/committed partitions diff --git a/pkg/daemon/ceph/osd/agent_test.go b/pkg/daemon/ceph/osd/agent_test.go index 7c5fc0de7..30b8849e5 100644 --- a/pkg/daemon/ceph/osd/agent_test.go +++ b/pkg/daemon/ceph/osd/agent_test.go @@ -120,7 +120,7 @@ func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig config.StoreConfig, execCount := 0 executor.MockExecuteCommand = func(debug bool, name string, command string, args ...string) error { - logger.Infof("RUN %d: %s %+v", execCount, command, args) + logger.Infof("EXEC %d: %s %+v", execCount, command, args) parts := strings.Split(name, " ") nameSuffix := parts[0] if len(parts) > 1 { @@ -247,15 +247,22 @@ func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig config.StoreConfig, "sdx": {Data: -1}, "sdy": {Data: -1}, }} - _, err = agent.configureAllDevices(context, devices) + _, err = agent.configureDevices(context, devices) assert.Nil(t, err) assert.Equal(t, int32(0), agent.configCounter) assert.Equal(t, 0, startCount) // 2 OSD procs should be started if !legacyProvisioner { - assert.Equal(t, 3, outputExecCount) - assert.Equal(t, 1, execCount) + if storeConfig.StoreType == config.Bluestore { + assert.Equal(t, 7, outputExecCount) + assert.Equal(t, 2, execCount) + } else { + assert.Equal(t, 6, outputExecCount) + // filestore on a device has two more calls than bluestore because of the mount/unmount commands of the legacy sdx device + // where sdy is created as the new c-v osd + assert.Equal(t, 4, execCount) + } } else if storeConfig.StoreType == config.Bluestore { assert.Equal(t, 12, outputExecCount) // Bluestore has 2 extra output exec calls to get device properties of each device to determine CRUSH weight assert.Equal(t, 5, execCount) // 1 osd mkfs for sdx, 3 partition steps for sdy, 1 osd mkfs for sdy @@ -374,8 +381,19 @@ func createTestAgent(t *testing.T, devices, configDir, nodeName string, storeCon executor := &exectest.MockExecutor{ MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) return "{\"key\":\"mysecurekey\", \"osdid\":3.0}", nil }, + MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, args ...string) (string, error) { + logger.Infof("%s %v", command, args) + if command == "ceph-volume" { + if len(args) == 3 && args[0] == "lvm" && args[1] == "batch" && args[2] == "--prepare" { + logger.Infof("test c-v not supported") + return "", fmt.Errorf("c-v not supported") + } + } + return "", nil + }, } cluster := &cephconfig.ClusterInfo{Name: "myclust"} context := &clusterd.Context{ConfigDir: configDir, Executor: executor, Clientset: testop.New(1)} @@ -449,7 +467,7 @@ func TestGetPartitionPerfScheme(t *testing.T) { devices, err := getAvailableDevices(context, []DesiredDevice{{Name: "sda"}, {Name: "sdb"}}, "sdc") assert.Nil(t, err) - scheme, err := a.getPartitionPerfScheme(context, devices) + scheme, _, err := a.getPartitionPerfScheme(context, devices, false) assert.Nil(t, err) assert.Equal(t, 2, len(scheme.Entries)) @@ -525,7 +543,7 @@ func TestGetPartitionSchemeDiskInUse(t *testing.T) { // get the partition scheme based on the available devices. Since sda is already in use, the partition // scheme returned should reflect that. devices, err := getAvailableDevices(context, []DesiredDevice{{Name: "sda"}}, "") - scheme, err := a.getPartitionPerfScheme(context, devices) + scheme, _, err := a.getPartitionPerfScheme(context, devices, false) assert.Nil(t, err) // the partition scheme should have a single entry for osd 1 on sda and it should have collocated data and metadata @@ -593,13 +611,30 @@ func TestGetPartitionSchemeDiskNameChanged(t *testing.T) { // get the current partition scheme. This should notice that the device names changed and update the // partition scheme to have the latest device names devices, err := getAvailableDevices(context, []DesiredDevice{{Name: "sda-changed"}}, "nvme01") - scheme, err := a.getPartitionPerfScheme(context, devices) + scheme, _, err := a.getPartitionPerfScheme(context, devices, false) assert.Nil(t, err) require.NotNil(t, scheme) assert.Equal(t, "sda-changed", scheme.Entries[0].Partitions[config.BlockPartitionType].Device) assert.Equal(t, "nvme01", scheme.Metadata.Device) assert.Equal(t, "nvme01", scheme.Entries[0].Partitions[config.WalPartitionType].Device) assert.Equal(t, "nvme01", scheme.Entries[0].Partitions[config.DatabasePartitionType].Device) + + // new devices should be skipped for ceph-volume to configure. + logger.Infof("testing skipping new devices that should be configured by ceph-volume instead of with legacy") + context.Devices = []*sys.LocalDisk{ + {Name: "sdx"}, + } + devices = &DeviceOsdMapping{Entries: map[string]*DeviceOsdIDEntry{"nvme05": {Data: -1, Config: DesiredDevice{Name: "nvme05", OSDsPerDevice: 5}}}} + scheme, skipped, err := a.getPartitionPerfScheme(context, devices, true) + assert.Nil(t, err) + require.NotNil(t, scheme) + require.Equal(t, 1, len(skipped.Entries)) + require.Equal(t, 1, len(scheme.Entries)) + assert.Equal(t, "nvme05", skipped.Entries["nvme05"].Config.Name) + assert.Equal(t, 5, skipped.Entries["nvme05"].Config.OSDsPerDevice) + for _, p := range scheme.Entries[0].Partitions { + assert.NotEqual(t, "nvme05", p.Device) + } } func TestPrepareOSDRoot(t *testing.T) { diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 3aad6a621..152ccd581 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -125,7 +125,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent) error { // start the desired OSDs on devices logger.Infof("configuring osd devices: %+v", devices) - deviceOSDs, err := agent.configureAllDevices(context, devices) + deviceOSDs, err := agent.configureDevices(context, devices) if err != nil { return fmt.Errorf("failed to configure devices. %+v", err) } diff --git a/pkg/daemon/ceph/osd/device_test.go b/pkg/daemon/ceph/osd/device_test.go index 5f2d0366a..12cbfaeef 100644 --- a/pkg/daemon/ceph/osd/device_test.go +++ b/pkg/daemon/ceph/osd/device_test.go @@ -73,7 +73,7 @@ func TestOSDBootstrap(t *testing.T) { err := createOSDBootstrapKeyring(context, clusterName, configDir) assert.Nil(t, err) - targetPath := path.Join(configDir, "bootstrap-osd/ceph.keyring") + targetPath := path.Join(configDir, bootstrapOsdKeyring) contents, err := ioutil.ReadFile(targetPath) assert.Nil(t, err) assert.NotEqual(t, -1, strings.Index(string(contents), "[client.bootstrap-osd]")) diff --git a/pkg/daemon/ceph/osd/init.go b/pkg/daemon/ceph/osd/init.go index ca97ae69b..9807d851a 100644 --- a/pkg/daemon/ceph/osd/init.go +++ b/pkg/daemon/ceph/osd/init.go @@ -29,7 +29,8 @@ import ( ) const ( - keyringFileName = "keyring" + keyringFileName = "keyring" + bootstrapOsdKeyring = "bootstrap-osd/ceph.keyring" ) // get the bootstrap OSD root dir @@ -64,7 +65,7 @@ func getOSDTempMonMapPath(osdDataPath string) string { // create a keyring for the bootstrap-osd client, it gets a limited set of privileges func createOSDBootstrapKeyring(context *clusterd.Context, clusterName, rootDir string) error { username := "client.bootstrap-osd" - keyringPath := path.Join(rootDir, "bootstrap-osd/ceph.keyring") + keyringPath := path.Join(rootDir, bootstrapOsdKeyring) access := []string{"mon", "allow profile bootstrap-osd"} keyringEval := func(key string) string { return fmt.Sprintf(bootstrapOSDKeyringTemplate, key) diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 8ad2c0cd2..88cbe09b5 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -37,37 +37,30 @@ const ( cephVolumeCmd = "ceph-volume" ) -func (a *OsdAgent) configureDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, bool, error) { +func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *DeviceOsdMapping) ([]oposd.OSDInfo, error) { var osds []oposd.OSDInfo - useCephVolume, err := getCephVolumeSupported(context) - if err != nil { - return nil, false, fmt.Errorf("failed to detect if ceph-volume is available. %+v", err) - } - if !useCephVolume { - return osds, false, nil - } - - if devices == nil || len(devices.Entries) == 0 { + var err error + if len(devices.Entries) == 0 { logger.Infof("no new devices to configure. returning devices already configured with ceph-volume.") osds, err = getCephVolumeOSDs(context, a.cluster.Name) if err != nil { logger.Infof("failed to get devices already provisioned by ceph-volume. %+v", err) } - return osds, true, nil + return osds, nil } err = createOSDBootstrapKeyring(context, a.cluster.Name, cephConfigDir) if err != nil { - return nil, true, fmt.Errorf("failed to generate osd keyring. %+v", err) + return nil, fmt.Errorf("failed to generate osd keyring. %+v", err) } if err = a.initializeDevices(context, devices); err != nil { - return nil, true, fmt.Errorf("failed to initialize devices. %+v", err) + return nil, fmt.Errorf("failed to initialize devices. %+v", err) } osds, err = getCephVolumeOSDs(context, a.cluster.Name) - return osds, true, err + return osds, err } func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceOsdMapping) error { @@ -86,6 +79,10 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO strconv.Itoa(a.storeConfig.OSDsPerDevice), }...) + // ceph-volume is soon implementing a parameter to specify the "fast devices", which correspond to the "metadataDevice" from the + // crd spec. After that is implemented, we can implement this. In the meantime, we fall back to use rook's partitioning. + metadataDeviceSpecified := false + configured := 0 for name, device := range devices.Entries { if device.LegacyPartitionsFound { @@ -96,7 +93,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO if device.Data == -1 { logger.Infof("configuring new device %s", name) deviceArg := path.Join("/dev", name) - if device.Config.OSDsPerDevice <= 1 { + if metadataDeviceSpecified { // the device will be configured as a batch at the end of the method batchArgs = append(batchArgs, deviceArg) configured++ @@ -135,10 +132,9 @@ func getCephVolumeSupported(context *clusterd.Context) (bool, error) { logger.Infof("supported version of ceph-volume not available") return false, nil } - logger.Warningf("unknown return code from ceph-volume when checking for compatibility: %d", exitStatus) + return false, fmt.Errorf("unknown return code from ceph-volume when checking for compatibility: %d", exitStatus) } - logger.Warningf("unknown ceph-volume failure. %+v", err) - return false, nil + return false, fmt.Errorf("unknown ceph-volume failure. %+v", err) } return true, nil @@ -188,7 +184,7 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string) ([]oposd.O osds = append(osds, osd) } - logger.Infof("%d osd devices configured on this node", len(osds)) + logger.Infof("%d ceph-volume osd devices configured on this node", len(osds)) return osds, nil } diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 8d620ba3c..79081aa4b 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -189,7 +189,7 @@ func (c *ClusterController) onAdd(obj interface{}) { if !cluster.Spec.CephVersion.AllowUnsupported { if !versionSupported(cluster.Spec.CephVersion.Name) { - logger.Errorf("unsupported ceph version detected: %s. allowUnupported must be set to true to run with this version.", cluster.Spec.CephVersion.Name) + logger.Errorf("unsupported ceph version detected: %s. allowUnsupported must be set to true to run with this version.", cluster.Spec.CephVersion.Name) return } } @@ -321,8 +321,10 @@ func (c *ClusterController) onUpdate(oldObj, newObj interface{}) { // if the image changed, we need to detect the new image version if oldClust.Spec.CephVersion.Image != newClust.Spec.CephVersion.Image { logger.Infof("the ceph version changed. detecting the new image version...") + cluster.Spec.CephVersion.Image = newClust.Spec.CephVersion.Image if err := cluster.setCephMajorVersion(15 * time.Minute); err != nil { logger.Errorf("unknown ceph major version. %+v", err) + cluster.Spec.CephVersion.Image = oldClust.Spec.CephVersion.Image return } } else {