From bdc3cf81468e442e7a7b7b2a2c31368f113461b4 Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 26 Feb 2019 16:11:25 -0700 Subject: [PATCH] osd: fix the device filter and improve device provisioning reliability All devices detected by the discovery pod were being passed to the OSD provisioning pod thus not always honoring the desired device list that should be provisioned. Now the provisioning pod will be given the desired state from the crd, then apply that state depending on the actual devices detected. Also added a helper to ensure OSDsPerDevice is always valid. Signed-off-by: travisn --- cmd/rook/ceph/osd.go | 2 +- pkg/daemon/ceph/osd/daemon.go | 25 ++++++++--- pkg/daemon/ceph/osd/device.go | 4 +- pkg/daemon/ceph/osd/volume.go | 12 ++++- pkg/daemon/ceph/osd/volume_test.go | 7 +++ pkg/operator/ceph/cluster/controller.go | 2 - pkg/operator/ceph/cluster/osd/osd.go | 52 +++++----------------- pkg/operator/ceph/cluster/osd/osd_test.go | 39 +++------------- pkg/operator/ceph/cluster/osd/spec.go | 2 +- pkg/operator/ceph/cluster/osd/spec_test.go | 8 ++-- pkg/operator/ceph/cluster/osd/status.go | 2 - pkg/operator/discover/discover.go | 38 ---------------- pkg/operator/discover/discover_test.go | 25 ----------- pkg/operator/edgefs/cluster/controller.go | 2 - pkg/util/sys/device.go | 14 ++++-- tests/framework/installer/device.go | 2 +- 16 files changed, 71 insertions(+), 165 deletions(-) diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 5cbb8ae0e..a7c2487ce 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -245,7 +245,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error { } dataDevices = []osddaemon.DesiredDevice{ - {Name: osdDataDeviceFilter, IsFilter: true}, + {Name: osdDataDeviceFilter, IsFilter: true, OSDsPerDevice: cfg.storeConfig.OSDsPerDevice}, } } else { var err error diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 1cc517dae..e7fbd7503 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -186,7 +186,7 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi if device.Type == sys.PartType { continue } - ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name) + partCount, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name) if err != nil { return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err) } @@ -197,12 +197,13 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi continue } + var deviceInfo *DeviceOsdIDEntry if metadataDevice != "" && metadataDevice == device.Name { // current device is desired as the metadata device - available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}, LegacyPartitionsFound: ownPartitions} + deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}} } else if len(desiredDevices) == 1 && desiredDevices[0].Name == "all" { // user has specified all devices, use the current one for data - available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, LegacyPartitionsFound: ownPartitions} + deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID} } else if len(desiredDevices) > 0 { var matched bool var err error @@ -211,8 +212,13 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi if desiredDevice.IsFilter { // the desired devices is a regular expression matched, err = regexp.Match(desiredDevice.Name, []byte(device.Name)) - } - if device.Name == desiredDevice.Name { + if err != nil { + logger.Errorf("regex failed on device %s and filter %s. %+v", device.Name, desiredDevice.Name, err) + continue + } + logger.Infof("device %s matches device filter %s: %t", device.Name, desiredDevice.Name, matched) + } else if device.Name == desiredDevice.Name { + logger.Infof("%s found in the desired devices", device.Name) matched = true } matchedDevice = desiredDevice @@ -223,13 +229,20 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi if err == nil && matched { // the current device matches the user specifies filter/list, use it for data - available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Config: matchedDevice} + deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID, Config: matchedDevice} } else { logger.Infof("skipping device %s that does not match the device filter/list (%v). %+v", device.Name, desiredDevices, err) } } else { logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name) } + + if deviceInfo != nil { + if partCount > 0 { + deviceInfo.LegacyPartitionsFound = ownPartitions + } + available.Entries[device.Name] = deviceInfo + } } return available, nil diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index 9542c8fe7..17b305d68 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -110,7 +110,7 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool } // check if partitions belong to rook - ownPartitions, devFS, err := sys.CheckIfDeviceAvailable(context.Executor, dataDetails.Device) + _, ownPartitions, devFS, err := sys.CheckIfDeviceAvailable(context.Executor, dataDetails.Device) if err != nil { return nil, fmt.Errorf("failed to format device. %+v", err) } @@ -170,7 +170,7 @@ func partitionMetadata(context *clusterd.Context, info *config.MetadataDeviceInf } // check one last time to make sure it's OK for us to format this metadata device - ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, info.Device) + _, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, info.Device) if err != nil { return fmt.Errorf("failed to get metadata device %s info: %+v", info.Device, err) } else if fs != "" || !ownPartitions { diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 88cbe09b5..edef2202f 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -76,7 +76,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO batchArgs := append(baseArgs, []string{ osdsPerDeviceFlag, - strconv.Itoa(a.storeConfig.OSDsPerDevice), + sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice), }...) // ceph-volume is soon implementing a parameter to specify the "fast devices", which correspond to the "metadataDevice" from the @@ -102,7 +102,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO immediateExecuteArgs := append(baseArgs, []string{ deviceArg, osdsPerDeviceFlag, - strconv.Itoa(device.Config.OSDsPerDevice), + sanitizeOSDsPerDevice(device.Config.OSDsPerDevice), }...) if err := context.Executor.ExecuteCommand(false, "", cephVolumeCmd, immediateExecuteArgs...); err != nil { @@ -123,6 +123,14 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO return nil } + +func sanitizeOSDsPerDevice(count int) string { + if count < 1 { + count = 1 + } + return strconv.Itoa(count) +} + func getCephVolumeSupported(context *clusterd.Context) (bool, error) { _, err := context.Executor.ExecuteCommandWithOutput(false, "", cephVolumeCmd, "lvm", "batch", "--prepare") if err != nil { diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 2a4f32e5a..32915ccbf 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -107,3 +107,10 @@ func TestParseCephVolumeResult(t *testing.T) { require.NotNil(t, osds) assert.Equal(t, 2, len(osds)) } + +func TestSanitizeOSDsPerDevice(t *testing.T) { + assert.Equal(t, "1", sanitizeOSDsPerDevice(-1)) + assert.Equal(t, "1", sanitizeOSDsPerDevice(0)) + assert.Equal(t, "1", sanitizeOSDsPerDevice(1)) + assert.Equal(t, "2", sanitizeOSDsPerDevice(2)) +} diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 5fc75f419..8d2780d0a 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -37,7 +37,6 @@ import ( "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/pkg/operator/ceph/object/user" "github.com/rook/rook/pkg/operator/ceph/pool" - "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/k8sutil" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -412,7 +411,6 @@ func (c *ClusterController) onDelete(obj interface{}) { if clust.Spec.Storage.AnyUseAllDevices() { c.devicesInUse = false } - discover.FreeDevicesByCluster(c.context, clust.Namespace) } func (c *ClusterController) handleDelete(cluster *cephv1.CephCluster, retryInterval time.Duration) error { diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index e461c913a..95c8d1b9b 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -174,13 +174,8 @@ func (c *Cluster) Start() error { logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes)) c.Storage.Nodes = validNodes - // orchestrate individual nodes, starting with any that are still ongoing (in the case that we - // are resuming a previous orchestration attempt) - config := newProvisionConfig() - logger.Infof("checking if orchestration is still in progress") - c.completeProvisionSkipOSDStart(config) - // start the jobs to provision the OSD devices and directories + config := newProvisionConfig() logger.Infof("start provisioning the osds on nodes, if needed") c.startProvisioning(config) @@ -202,7 +197,10 @@ func (c *Cluster) Start() error { } func (c *Cluster) startProvisioning(config *provisionConfig) { - config.devicesToUse = make(map[string][]rookalpha.Device) + if len(c.dataDirHostPath) == 0 { + logger.Warningf("skipping osd provisioning where no dataDirHostPath is set") + return + } // start with nodes currently in the storage spec for _, node := range c.Storage.Nodes { @@ -224,23 +222,11 @@ func (c *Cluster) startProvisioning(config *provisionConfig) { config.addError("failed to set orchestration starting status for node %s: %+v", n.Name, err) continue } - config.devicesToUse[n.Name] = n.Devices - availDev, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace, n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices()) - if deviceErr != nil { - logger.Warningf("failed to get devices for node %s cluster %s: %v", n.Name, c.Namespace, deviceErr) - } else { - config.devicesToUse[n.Name] = availDev - logger.Infof("avail devices for node %s: %+v", n.Name, availDev) - } - if len(availDev) == 0 && len(c.dataDirHostPath) == 0 { - config.addError("empty volumes for node %s", n.Name) - continue - } // create the job that prepares osds on the node storeConfig := osdconfig.ToStoreConfig(n.Config) metadataDevice := osdconfig.MetadataDevice(n.Config) - job, err := c.makeJob(n.Name, config.devicesToUse[n.Name], n.Selection, n.Resources, storeConfig, metadataDevice, n.Location) + job, err := c.makeJob(n.Name, n.Devices, n.Selection, n.Resources, storeConfig, metadataDevice, n.Location) if err != nil { message := fmt.Sprintf("failed to create prepare job node %s: %v", n.Name, err) config.addError(message) @@ -252,8 +238,9 @@ func (c *Cluster) startProvisioning(config *provisionConfig) { } if !c.runJob(job, n.Name, config, "provision") { - if err = discover.FreeDevices(c.context, n.Name, c.Namespace); err != nil { - logger.Warningf("failed to free devices: %s", err) + status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: fmt.Sprintf("failed to start osd provisioning on node %s", n.Name)} + if err := c.updateNodeStatus(n.Name, status); err != nil { + config.addError("failed to update node %s status. %+v", n.Name, err) } } } @@ -293,30 +280,18 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig // start osds for _, osd := range osds { logger.Debugf("start osd %v", osd) - dp, err := c.makeDeployment(n.Name, config.devicesToUse[n.Name], n.Selection, n.Resources, storeConfig, metadataDevice, n.Location, osd) + dp, err := c.makeDeployment(n.Name, n.Selection, n.Resources, storeConfig, metadataDevice, n.Location, osd) if err != nil { - errMsg := fmt.Sprintf("nil deployment for node %s: %v", n.Name, err) + errMsg := fmt.Sprintf("failed to create deployment for node %s: %v", n.Name, err) config.addError(errMsg) - err = discover.FreeDevices(c.context, n.Name, c.Namespace) - if err != nil { - logger.Warningf("failed to free devices: %s", err) - } continue } - if err = c.deleteDeploymentWithLegacyName(osd.ID); err != nil { - logger.Warningf("failed to delete legacy osd deployment. %+v", err) - } - _, err = c.context.Clientset.Apps().Deployments(c.Namespace).Create(dp) if err != nil { if !errors.IsAlreadyExists(err) { // we failed to create job, update the orchestration status for this node logger.Warningf("failed to create osd deployment for node %s, osd %v: %+v", n.Name, osd, err) - err = discover.FreeDevices(c.context, n.Name, c.Namespace) - if err != nil { - logger.Warningf("failed to free devices: %s", err) - } continue } logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID) @@ -329,11 +304,6 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig } } -func (c *Cluster) deleteDeploymentWithLegacyName(osdID int) error { - legacyName := fmt.Sprintf(legacyAppNameFmt, osdID) - return k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, legacyName) -} - func (c *Cluster) handleRemovedNodes(config *provisionConfig) { // find all removed nodes (if any) and start orchestration to remove them from the cluster removedNodes, err := c.findRemovedNodes() diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index f69c291f7..21230fab1 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -88,35 +88,6 @@ func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake return err } -func TestLegacyDeployment(t *testing.T) { - clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", cephv1.CephVersionSpec{}, - rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - - osdID := 23 - d := &apps.Deployment{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf(legacyAppNameFmt, osdID), Namespace: c.Namespace}} - _, err := clientset.Apps().Deployments(c.Namespace).Create(d) - require.Nil(t, err) - - // delete the deployment - assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID)) - deployments, err := clientset.Apps().Deployments(c.Namespace).List(metav1.ListOptions{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(deployments.Items)) - - // return success if the deployment doesn't exist - assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID)) - - // don't delete the newer deployment name - d = &apps.Deployment{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf(osdAppNameFmt, osdID), Namespace: c.Namespace}} - _, err = clientset.Apps().Deployments(c.Namespace).Create(d) - require.Nil(t, err) - assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID)) - deployments, err = clientset.Apps().Deployments(c.Namespace).List(metav1.ListOptions{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(deployments.Items)) -} - func TestAddRemoveNode(t *testing.T) { // create a storage spec with the given nodes/devices/dirs nodeName := "node8230" @@ -146,7 +117,7 @@ func TestAddRemoveNode(t *testing.T) { clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(statusMapWatcher, nil)) c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1.CephVersionSpec{}, - storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) + storageSpec, "/foo", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // kick off the start of the orchestration in a goroutine var startErr error @@ -275,17 +246,17 @@ func TestDiscoverOSDs(t *testing.T) { node2 := "n2" osd1 := OSDInfo{ID: 0, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"} - d1, err := c.makeDeployment(node1, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd1) + d1, err := c.makeDeployment(node1, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd1) assert.Nil(t, err) assert.NotNil(t, d1) osd2 := OSDInfo{ID: 101, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"} - d2, err := c.makeDeployment(node1, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd2) + d2, err := c.makeDeployment(node1, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd2) assert.Nil(t, err) assert.NotNil(t, d2) osd3 := OSDInfo{ID: 23, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"} - d3, err := c.makeDeployment(node2, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd3) + d3, err := c.makeDeployment(node2, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd3) assert.Nil(t, err) assert.NotNil(t, d3) @@ -338,7 +309,7 @@ func TestAddNodeFailure(t *testing.T) { assert.Nil(t, cmErr) c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1.CephVersionSpec{}, - storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) + storageSpec, "/foo", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // kick off the start of the orchestration in a goroutine var startErr error diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 4cdfc05cd..a8f401332 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -76,7 +76,7 @@ func (c *Cluster) makeJob(nodeName string, devices []rookalpha.Device, return job, nil } -func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, +func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, resources v1.ResourceRequirements, storeConfig config.StoreConfig, metadataDevice, location string, osd OSDInfo) (*apps.Deployment, error) { replicaCount := int32(1) diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 99ed3324f..a4d397b3e 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -86,7 +86,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { ID: 0, } - deployment, err := c.makeDeployment(n.Name, devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + deployment, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.Nil(t, err) assert.NotNil(t, deployment) assert.Equal(t, "rook-ceph-osd-0", deployment.Name) @@ -165,7 +165,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { IsDirectory: true, DataPath: "/my/root/path/osd1", } - deployment, err := c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + deployment, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.NotNil(t, deployment) assert.Nil(t, err) // pod spec should have a volume for the given dir in the main container and the init container @@ -192,7 +192,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { IsDirectory: true, DataPath: "/var/lib/rook/osd1", } - deployment, err = c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + deployment, err = c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.NotNil(t, deployment) assert.Nil(t, err) // pod spec should have a volume for the given dir in the main container and the init container @@ -300,7 +300,7 @@ func TestHostNetwork(t *testing.T) { osd := OSDInfo{ ID: 0, } - r, err := c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + r, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.NotNil(t, r) assert.Nil(t, err) diff --git a/pkg/operator/ceph/cluster/osd/status.go b/pkg/operator/ceph/cluster/osd/status.go index c4512cbc2..a84f6cd24 100644 --- a/pkg/operator/ceph/cluster/osd/status.go +++ b/pkg/operator/ceph/cluster/osd/status.go @@ -22,7 +22,6 @@ import ( "fmt" "time" - rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" apps "k8s.io/api/apps/v1" @@ -48,7 +47,6 @@ const ( ) type provisionConfig struct { - devicesToUse map[string][]rookalpha.Device errorMessages []string } diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index 0f8397853..ee5f06c29 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -298,44 +298,6 @@ func ListDevicesInUse(context *clusterd.Context, namespace, nodeName string) ([] return devices, nil } -// FreeDevices frees up devices used by a cluster on a node. -func FreeDevices(context *clusterd.Context, nodeName, clusterName string) error { - if len(nodeName) == 0 || len(clusterName) == 0 { - return nil - } - namespace := os.Getenv(k8sutil.PodNamespaceEnvVar) - cmName := k8sutil.TruncateNodeName(fmt.Sprintf(deviceInUseCMName, clusterName, "%s"), nodeName) - // delete configmap - err := context.Clientset.CoreV1().ConfigMaps(namespace).Delete(cmName, &metav1.DeleteOptions{}) - if err != nil && !kserrors.IsNotFound(err) { - return fmt.Errorf("failed to delete configmap %s/%s. %+v", namespace, cmName, err) - } - - return nil -} - -// FreeDevicesByCluster frees devices on all nodes that are used by the cluster -func FreeDevicesByCluster(context *clusterd.Context, clusterName string) error { - logger.Infof("freeing devices used by cluster %s", clusterName) - namespace := os.Getenv(k8sutil.PodNamespaceEnvVar) - listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", deviceInUseClusterAttr, clusterName)} - cms, err := context.Clientset.CoreV1().ConfigMaps(namespace).List(listOpts) - if err != nil { - return fmt.Errorf("failed to list device in use configmaps for cluster %s: %+v", clusterName, err) - } - - for _, cm := range cms.Items { - // delete configmap - cmName := cm.Name - err := context.Clientset.CoreV1().ConfigMaps(namespace).Delete(cmName, &metav1.DeleteOptions{}) - logger.Infof("deleting configmap %s: %+v", cmName, err) - if err != nil { - return err - } - } - return nil -} - // GetAvailableDevices conducts outer join using input filters with free devices that a node has. It marks the devices from join result as in-use. func GetAvailableDevices(context *clusterd.Context, nodeName, clusterName string, devices []rookalpha.Device, filter string, useAllDevices bool) ([]rookalpha.Device, error) { results := []rookalpha.Device{} diff --git a/pkg/operator/discover/discover_test.go b/pkg/operator/discover/discover_test.go index 8e8f1a5f1..45f71cb9c 100644 --- a/pkg/operator/discover/discover_test.go +++ b/pkg/operator/discover/discover_test.go @@ -116,29 +116,4 @@ func TestGetAvailableDevices(t *testing.T) { devices, err = GetAvailableDevices(context, nodeName, ns, d, "^sd.", false) assert.Nil(t, err) assert.Equal(t, 1, len(devices)) - - err = FreeDevices(context, nodeName, ns) - assert.Nil(t, err) - // all devices freed - devices, err = GetAvailableDevices(context, nodeName, ns, nil, "^sd.", false) - assert.Nil(t, err) - assert.Equal(t, 4, len(devices)) - // devices should be in use now, 2nd try gets the same list - devices, err = GetAvailableDevices(context, nodeName, ns, nil, "^sd.", false) - assert.Nil(t, err) - assert.Equal(t, 4, len(devices)) - - err = FreeDevices(context, nodeName, ns) - assert.Nil(t, err) - - devices, err = GetAvailableDevices(context, nodeName, ns, nil, "", true) - assert.Nil(t, err) - assert.Equal(t, 5, len(devices)) - // devices should be in use now, 2nd try gets the same list - devices, err = GetAvailableDevices(context, nodeName, ns, nil, "", true) - assert.Nil(t, err) - assert.Equal(t, 5, len(devices)) - - err = FreeDevices(context, nodeName, ns) - assert.Nil(t, err) } diff --git a/pkg/operator/edgefs/cluster/controller.go b/pkg/operator/edgefs/cluster/controller.go index 15999f79a..997836e7e 100644 --- a/pkg/operator/edgefs/cluster/controller.go +++ b/pkg/operator/edgefs/cluster/controller.go @@ -28,7 +28,6 @@ import ( opkit "github.com/rook/operator-kit" edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/edgefs/iscsi" "github.com/rook/rook/pkg/operator/edgefs/isgw" "github.com/rook/rook/pkg/operator/edgefs/nfs" @@ -322,7 +321,6 @@ func (c *ClusterController) onDelete(obj interface{}) { if clust.Spec.Storage.AnyUseAllDevices() { c.devicesInUse = false } - discover.FreeDevicesByCluster(c.context, clust.Namespace) } func (c *ClusterController) handleDelete(clust *edgefsv1alpha1.Cluster, retryInterval time.Duration) error { diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go index 75d4c5403..2066ea492 100644 --- a/pkg/util/sys/device.go +++ b/pkg/util/sys/device.go @@ -289,12 +289,18 @@ func UnmountDevice(devicePath string, executor exec.Executor) error { return nil } -func CheckIfDeviceAvailable(executor exec.Executor, name string) (bool, string, error) { +// CheckIfDeviceAvailable checks if a device is available for consumption. The caller +// needs to decide based on the return values whether it is available. The return values are +// the number of partitions, whether Rook has created partitions on the device in the past +// possibly from the same or a previous cluster, the filesystem found, or an err if failed +// to retrieve the properties. +func CheckIfDeviceAvailable(executor exec.Executor, name string) (int, bool, string, error) { ownPartitions := true partitions, _, err := GetDevicePartitions(name, executor) if err != nil { - return false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err) + return 0, false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err) } + partCount := len(partitions) if !RookOwnsPartitions(partitions) { ownPartitions = false } @@ -302,10 +308,10 @@ func CheckIfDeviceAvailable(executor exec.Executor, name string) (bool, string, // check if there is a file system on the device devFS, err := GetDeviceFilesystems(name, executor) if err != nil { - return false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err) + return 0, false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err) } - return ownPartitions, devFS, nil + return partCount, ownPartitions, devFS, nil } // RookOwnsPartitions check if all partitions in list are owned by Rook diff --git a/tests/framework/installer/device.go b/tests/framework/installer/device.go index bf3e510cf..1ca6f772e 100644 --- a/tests/framework/installer/device.go +++ b/tests/framework/installer/device.go @@ -40,7 +40,7 @@ func IsAdditionalDeviceAvailableOnCluster() bool { continue } - ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device) + _, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device) if err != nil { logger.Warningf("failed to detect device %s availability. %+v", device, err) continue