diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 5cbb8ae0e..a7c2487ce 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -245,7 +245,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error { } dataDevices = []osddaemon.DesiredDevice{ - {Name: osdDataDeviceFilter, IsFilter: true}, + {Name: osdDataDeviceFilter, IsFilter: true, OSDsPerDevice: cfg.storeConfig.OSDsPerDevice}, } } else { var err error diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 1cc517dae..e7fbd7503 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -186,7 +186,7 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi if device.Type == sys.PartType { continue } - ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name) + partCount, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name) if err != nil { return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err) } @@ -197,12 +197,13 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi continue } + var deviceInfo *DeviceOsdIDEntry if metadataDevice != "" && metadataDevice == device.Name { // current device is desired as the metadata device - available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}, LegacyPartitionsFound: ownPartitions} + deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}} } else if len(desiredDevices) == 1 && desiredDevices[0].Name == "all" { // user has specified all devices, use the current one for data - available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, LegacyPartitionsFound: ownPartitions} + deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID} } else if len(desiredDevices) > 0 { var matched bool var err error @@ -211,8 +212,13 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi if desiredDevice.IsFilter { // the desired devices is a regular expression matched, err = regexp.Match(desiredDevice.Name, []byte(device.Name)) - } - if device.Name == desiredDevice.Name { + if err != nil { + logger.Errorf("regex failed on device %s and filter %s. %+v", device.Name, desiredDevice.Name, err) + continue + } + logger.Infof("device %s matches device filter %s: %t", device.Name, desiredDevice.Name, matched) + } else if device.Name == desiredDevice.Name { + logger.Infof("%s found in the desired devices", device.Name) matched = true } matchedDevice = desiredDevice @@ -223,13 +229,20 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi if err == nil && matched { // the current device matches the user specifies filter/list, use it for data - available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Config: matchedDevice} + deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID, Config: matchedDevice} } else { logger.Infof("skipping device %s that does not match the device filter/list (%v). %+v", device.Name, desiredDevices, err) } } else { logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name) } + + if deviceInfo != nil { + if partCount > 0 { + deviceInfo.LegacyPartitionsFound = ownPartitions + } + available.Entries[device.Name] = deviceInfo + } } return available, nil diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index 9542c8fe7..17b305d68 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -110,7 +110,7 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool } // check if partitions belong to rook - ownPartitions, devFS, err := sys.CheckIfDeviceAvailable(context.Executor, dataDetails.Device) + _, ownPartitions, devFS, err := sys.CheckIfDeviceAvailable(context.Executor, dataDetails.Device) if err != nil { return nil, fmt.Errorf("failed to format device. %+v", err) } @@ -170,7 +170,7 @@ func partitionMetadata(context *clusterd.Context, info *config.MetadataDeviceInf } // check one last time to make sure it's OK for us to format this metadata device - ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, info.Device) + _, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, info.Device) if err != nil { return fmt.Errorf("failed to get metadata device %s info: %+v", info.Device, err) } else if fs != "" || !ownPartitions { diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 88cbe09b5..edef2202f 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -76,7 +76,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO batchArgs := append(baseArgs, []string{ osdsPerDeviceFlag, - strconv.Itoa(a.storeConfig.OSDsPerDevice), + sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice), }...) // ceph-volume is soon implementing a parameter to specify the "fast devices", which correspond to the "metadataDevice" from the @@ -102,7 +102,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO immediateExecuteArgs := append(baseArgs, []string{ deviceArg, osdsPerDeviceFlag, - strconv.Itoa(device.Config.OSDsPerDevice), + sanitizeOSDsPerDevice(device.Config.OSDsPerDevice), }...) if err := context.Executor.ExecuteCommand(false, "", cephVolumeCmd, immediateExecuteArgs...); err != nil { @@ -123,6 +123,14 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO return nil } + +func sanitizeOSDsPerDevice(count int) string { + if count < 1 { + count = 1 + } + return strconv.Itoa(count) +} + func getCephVolumeSupported(context *clusterd.Context) (bool, error) { _, err := context.Executor.ExecuteCommandWithOutput(false, "", cephVolumeCmd, "lvm", "batch", "--prepare") if err != nil { diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 2a4f32e5a..32915ccbf 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -107,3 +107,10 @@ func TestParseCephVolumeResult(t *testing.T) { require.NotNil(t, osds) assert.Equal(t, 2, len(osds)) } + +func TestSanitizeOSDsPerDevice(t *testing.T) { + assert.Equal(t, "1", sanitizeOSDsPerDevice(-1)) + assert.Equal(t, "1", sanitizeOSDsPerDevice(0)) + assert.Equal(t, "1", sanitizeOSDsPerDevice(1)) + assert.Equal(t, "2", sanitizeOSDsPerDevice(2)) +} diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index e30203691..40271f06e 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -37,7 +37,6 @@ import ( "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/pkg/operator/ceph/object/user" "github.com/rook/rook/pkg/operator/ceph/pool" - "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/k8sutil" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -412,7 +411,6 @@ func (c *ClusterController) onDelete(obj interface{}) { if clust.Spec.Storage.AnyUseAllDevices() { c.devicesInUse = false } - discover.FreeDevicesByCluster(c.context, clust.Namespace) } func (c *ClusterController) handleDelete(cluster *cephv1.CephCluster, retryInterval time.Duration) error { diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index e67f7bb25..99d4a82d0 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -174,13 +174,8 @@ func (c *Cluster) Start() error { logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes)) c.Storage.Nodes = validNodes - // orchestrate individual nodes, starting with any that are still ongoing (in the case that we - // are resuming a previous orchestration attempt) - config := newProvisionConfig() - logger.Infof("checking if orchestration is still in progress") - c.completeProvisionSkipOSDStart(config) - // start the jobs to provision the OSD devices and directories + config := newProvisionConfig() logger.Infof("start provisioning the osds on nodes, if needed") c.startProvisioning(config) @@ -202,7 +197,10 @@ func (c *Cluster) Start() error { } func (c *Cluster) startProvisioning(config *provisionConfig) { - config.devicesToUse = make(map[string][]rookalpha.Device) + if len(c.dataDirHostPath) == 0 { + logger.Warningf("skipping osd provisioning where no dataDirHostPath is set") + return + } // start with nodes currently in the storage spec for _, node := range c.Storage.Nodes { @@ -224,23 +222,11 @@ func (c *Cluster) startProvisioning(config *provisionConfig) { config.addError("failed to set orchestration starting status for node %s: %+v", n.Name, err) continue } - config.devicesToUse[n.Name] = n.Devices - availDev, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace, n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices()) - if deviceErr != nil { - logger.Warningf("failed to get devices for node %s cluster %s: %v", n.Name, c.Namespace, deviceErr) - } else { - config.devicesToUse[n.Name] = availDev - logger.Infof("avail devices for node %s: %+v", n.Name, availDev) - } - if len(availDev) == 0 && len(c.dataDirHostPath) == 0 { - config.addError("empty volumes for node %s", n.Name) - continue - } // create the job that prepares osds on the node storeConfig := osdconfig.ToStoreConfig(n.Config) metadataDevice := osdconfig.MetadataDevice(n.Config) - job, err := c.makeJob(n.Name, config.devicesToUse[n.Name], n.Selection, n.Resources, storeConfig, metadataDevice, n.Location) + job, err := c.makeJob(n.Name, n.Devices, n.Selection, n.Resources, storeConfig, metadataDevice, n.Location) if err != nil { message := fmt.Sprintf("failed to create prepare job node %s: %v", n.Name, err) config.addError(message) @@ -252,8 +238,9 @@ func (c *Cluster) startProvisioning(config *provisionConfig) { } if !c.runJob(job, n.Name, config, "provision") { - if err = discover.FreeDevices(c.context, n.Name, c.Namespace); err != nil { - logger.Warningf("failed to free devices: %s", err) + status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: fmt.Sprintf("failed to start osd provisioning on node %s", n.Name)} + if err := c.updateNodeStatus(n.Name, status); err != nil { + config.addError("failed to update node %s status. %+v", n.Name, err) } } } @@ -293,30 +280,18 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig // start osds for _, osd := range osds { logger.Debugf("start osd %v", osd) - dp, err := c.makeDeployment(n.Name, config.devicesToUse[n.Name], n.Selection, n.Resources, storeConfig, metadataDevice, n.Location, osd) + dp, err := c.makeDeployment(n.Name, n.Selection, n.Resources, storeConfig, metadataDevice, n.Location, osd) if err != nil { - errMsg := fmt.Sprintf("nil deployment for node %s: %v", n.Name, err) + errMsg := fmt.Sprintf("failed to create deployment for node %s: %v", n.Name, err) config.addError(errMsg) - err = discover.FreeDevices(c.context, n.Name, c.Namespace) - if err != nil { - logger.Warningf("failed to free devices: %s", err) - } continue } - if err = c.deleteDeploymentWithLegacyName(osd.ID); err != nil { - logger.Warningf("failed to delete legacy osd deployment. %+v", err) - } - _, err = c.context.Clientset.Apps().Deployments(c.Namespace).Create(dp) if err != nil { if !errors.IsAlreadyExists(err) { // we failed to create job, update the orchestration status for this node logger.Warningf("failed to create osd deployment for node %s, osd %v: %+v", n.Name, osd, err) - err = discover.FreeDevices(c.context, n.Name, c.Namespace) - if err != nil { - logger.Warningf("failed to free devices: %s", err) - } continue } logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID) @@ -329,11 +304,6 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig } } -func (c *Cluster) deleteDeploymentWithLegacyName(osdID int) error { - legacyName := fmt.Sprintf(legacyAppNameFmt, osdID) - return k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, legacyName) -} - func (c *Cluster) handleRemovedNodes(config *provisionConfig) { // find all removed nodes (if any) and start orchestration to remove them from the cluster removedNodes, err := c.findRemovedNodes() diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index f69c291f7..21230fab1 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -88,35 +88,6 @@ func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake return err } -func TestLegacyDeployment(t *testing.T) { - clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", cephv1.CephVersionSpec{}, - rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) - - osdID := 23 - d := &apps.Deployment{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf(legacyAppNameFmt, osdID), Namespace: c.Namespace}} - _, err := clientset.Apps().Deployments(c.Namespace).Create(d) - require.Nil(t, err) - - // delete the deployment - assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID)) - deployments, err := clientset.Apps().Deployments(c.Namespace).List(metav1.ListOptions{}) - assert.Nil(t, err) - assert.Equal(t, 0, len(deployments.Items)) - - // return success if the deployment doesn't exist - assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID)) - - // don't delete the newer deployment name - d = &apps.Deployment{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf(osdAppNameFmt, osdID), Namespace: c.Namespace}} - _, err = clientset.Apps().Deployments(c.Namespace).Create(d) - require.Nil(t, err) - assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID)) - deployments, err = clientset.Apps().Deployments(c.Namespace).List(metav1.ListOptions{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(deployments.Items)) -} - func TestAddRemoveNode(t *testing.T) { // create a storage spec with the given nodes/devices/dirs nodeName := "node8230" @@ -146,7 +117,7 @@ func TestAddRemoveNode(t *testing.T) { clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(statusMapWatcher, nil)) c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1.CephVersionSpec{}, - storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) + storageSpec, "/foo", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // kick off the start of the orchestration in a goroutine var startErr error @@ -275,17 +246,17 @@ func TestDiscoverOSDs(t *testing.T) { node2 := "n2" osd1 := OSDInfo{ID: 0, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"} - d1, err := c.makeDeployment(node1, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd1) + d1, err := c.makeDeployment(node1, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd1) assert.Nil(t, err) assert.NotNil(t, d1) osd2 := OSDInfo{ID: 101, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"} - d2, err := c.makeDeployment(node1, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd2) + d2, err := c.makeDeployment(node1, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd2) assert.Nil(t, err) assert.NotNil(t, d2) osd3 := OSDInfo{ID: 23, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"} - d3, err := c.makeDeployment(node2, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd3) + d3, err := c.makeDeployment(node2, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd3) assert.Nil(t, err) assert.NotNil(t, d3) @@ -338,7 +309,7 @@ func TestAddNodeFailure(t *testing.T) { assert.Nil(t, cmErr) c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1.CephVersionSpec{}, - storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) + storageSpec, "/foo", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // kick off the start of the orchestration in a goroutine var startErr error diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index a90d0a436..a2b9c453c 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -76,7 +76,7 @@ func (c *Cluster) makeJob(nodeName string, devices []rookalpha.Device, return job, nil } -func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, +func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, resources v1.ResourceRequirements, storeConfig config.StoreConfig, metadataDevice, location string, osd OSDInfo) (*apps.Deployment, error) { replicaCount := int32(1) diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 99ed3324f..a4d397b3e 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -86,7 +86,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { ID: 0, } - deployment, err := c.makeDeployment(n.Name, devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + deployment, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.Nil(t, err) assert.NotNil(t, deployment) assert.Equal(t, "rook-ceph-osd-0", deployment.Name) @@ -165,7 +165,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { IsDirectory: true, DataPath: "/my/root/path/osd1", } - deployment, err := c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + deployment, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.NotNil(t, deployment) assert.Nil(t, err) // pod spec should have a volume for the given dir in the main container and the init container @@ -192,7 +192,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { IsDirectory: true, DataPath: "/var/lib/rook/osd1", } - deployment, err = c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + deployment, err = c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.NotNil(t, deployment) assert.Nil(t, err) // pod spec should have a volume for the given dir in the main container and the init container @@ -300,7 +300,7 @@ func TestHostNetwork(t *testing.T) { osd := OSDInfo{ ID: 0, } - r, err := c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) + r, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd) assert.NotNil(t, r) assert.Nil(t, err) diff --git a/pkg/operator/ceph/cluster/osd/status.go b/pkg/operator/ceph/cluster/osd/status.go index c4512cbc2..a84f6cd24 100644 --- a/pkg/operator/ceph/cluster/osd/status.go +++ b/pkg/operator/ceph/cluster/osd/status.go @@ -22,7 +22,6 @@ import ( "fmt" "time" - rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" apps "k8s.io/api/apps/v1" @@ -48,7 +47,6 @@ const ( ) type provisionConfig struct { - devicesToUse map[string][]rookalpha.Device errorMessages []string } diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index 0f8397853..ee5f06c29 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -298,44 +298,6 @@ func ListDevicesInUse(context *clusterd.Context, namespace, nodeName string) ([] return devices, nil } -// FreeDevices frees up devices used by a cluster on a node. -func FreeDevices(context *clusterd.Context, nodeName, clusterName string) error { - if len(nodeName) == 0 || len(clusterName) == 0 { - return nil - } - namespace := os.Getenv(k8sutil.PodNamespaceEnvVar) - cmName := k8sutil.TruncateNodeName(fmt.Sprintf(deviceInUseCMName, clusterName, "%s"), nodeName) - // delete configmap - err := context.Clientset.CoreV1().ConfigMaps(namespace).Delete(cmName, &metav1.DeleteOptions{}) - if err != nil && !kserrors.IsNotFound(err) { - return fmt.Errorf("failed to delete configmap %s/%s. %+v", namespace, cmName, err) - } - - return nil -} - -// FreeDevicesByCluster frees devices on all nodes that are used by the cluster -func FreeDevicesByCluster(context *clusterd.Context, clusterName string) error { - logger.Infof("freeing devices used by cluster %s", clusterName) - namespace := os.Getenv(k8sutil.PodNamespaceEnvVar) - listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", deviceInUseClusterAttr, clusterName)} - cms, err := context.Clientset.CoreV1().ConfigMaps(namespace).List(listOpts) - if err != nil { - return fmt.Errorf("failed to list device in use configmaps for cluster %s: %+v", clusterName, err) - } - - for _, cm := range cms.Items { - // delete configmap - cmName := cm.Name - err := context.Clientset.CoreV1().ConfigMaps(namespace).Delete(cmName, &metav1.DeleteOptions{}) - logger.Infof("deleting configmap %s: %+v", cmName, err) - if err != nil { - return err - } - } - return nil -} - // GetAvailableDevices conducts outer join using input filters with free devices that a node has. It marks the devices from join result as in-use. func GetAvailableDevices(context *clusterd.Context, nodeName, clusterName string, devices []rookalpha.Device, filter string, useAllDevices bool) ([]rookalpha.Device, error) { results := []rookalpha.Device{} diff --git a/pkg/operator/discover/discover_test.go b/pkg/operator/discover/discover_test.go index 8e8f1a5f1..45f71cb9c 100644 --- a/pkg/operator/discover/discover_test.go +++ b/pkg/operator/discover/discover_test.go @@ -116,29 +116,4 @@ func TestGetAvailableDevices(t *testing.T) { devices, err = GetAvailableDevices(context, nodeName, ns, d, "^sd.", false) assert.Nil(t, err) assert.Equal(t, 1, len(devices)) - - err = FreeDevices(context, nodeName, ns) - assert.Nil(t, err) - // all devices freed - devices, err = GetAvailableDevices(context, nodeName, ns, nil, "^sd.", false) - assert.Nil(t, err) - assert.Equal(t, 4, len(devices)) - // devices should be in use now, 2nd try gets the same list - devices, err = GetAvailableDevices(context, nodeName, ns, nil, "^sd.", false) - assert.Nil(t, err) - assert.Equal(t, 4, len(devices)) - - err = FreeDevices(context, nodeName, ns) - assert.Nil(t, err) - - devices, err = GetAvailableDevices(context, nodeName, ns, nil, "", true) - assert.Nil(t, err) - assert.Equal(t, 5, len(devices)) - // devices should be in use now, 2nd try gets the same list - devices, err = GetAvailableDevices(context, nodeName, ns, nil, "", true) - assert.Nil(t, err) - assert.Equal(t, 5, len(devices)) - - err = FreeDevices(context, nodeName, ns) - assert.Nil(t, err) } diff --git a/pkg/operator/edgefs/cluster/controller.go b/pkg/operator/edgefs/cluster/controller.go index 15999f79a..997836e7e 100644 --- a/pkg/operator/edgefs/cluster/controller.go +++ b/pkg/operator/edgefs/cluster/controller.go @@ -28,7 +28,6 @@ import ( opkit "github.com/rook/operator-kit" edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/edgefs/iscsi" "github.com/rook/rook/pkg/operator/edgefs/isgw" "github.com/rook/rook/pkg/operator/edgefs/nfs" @@ -322,7 +321,6 @@ func (c *ClusterController) onDelete(obj interface{}) { if clust.Spec.Storage.AnyUseAllDevices() { c.devicesInUse = false } - discover.FreeDevicesByCluster(c.context, clust.Namespace) } func (c *ClusterController) handleDelete(clust *edgefsv1alpha1.Cluster, retryInterval time.Duration) error { diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go index 75d4c5403..2066ea492 100644 --- a/pkg/util/sys/device.go +++ b/pkg/util/sys/device.go @@ -289,12 +289,18 @@ func UnmountDevice(devicePath string, executor exec.Executor) error { return nil } -func CheckIfDeviceAvailable(executor exec.Executor, name string) (bool, string, error) { +// CheckIfDeviceAvailable checks if a device is available for consumption. The caller +// needs to decide based on the return values whether it is available. The return values are +// the number of partitions, whether Rook has created partitions on the device in the past +// possibly from the same or a previous cluster, the filesystem found, or an err if failed +// to retrieve the properties. +func CheckIfDeviceAvailable(executor exec.Executor, name string) (int, bool, string, error) { ownPartitions := true partitions, _, err := GetDevicePartitions(name, executor) if err != nil { - return false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err) + return 0, false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err) } + partCount := len(partitions) if !RookOwnsPartitions(partitions) { ownPartitions = false } @@ -302,10 +308,10 @@ func CheckIfDeviceAvailable(executor exec.Executor, name string) (bool, string, // check if there is a file system on the device devFS, err := GetDeviceFilesystems(name, executor) if err != nil { - return false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err) + return 0, false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err) } - return ownPartitions, devFS, nil + return partCount, ownPartitions, devFS, nil } // RookOwnsPartitions check if all partitions in list are owned by Rook diff --git a/tests/framework/installer/device.go b/tests/framework/installer/device.go index bf3e510cf..1ca6f772e 100644 --- a/tests/framework/installer/device.go +++ b/tests/framework/installer/device.go @@ -40,7 +40,7 @@ func IsAdditionalDeviceAvailableOnCluster() bool { continue } - ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device) + _, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device) if err != nil { logger.Warningf("failed to detect device %s availability. %+v", device, err) continue