Merge pull request #2703 from travisn/osds-per-device-with-filter
Clean up provisioning of OSDs on devices
This commit is contained in:
16 files changed
+71
-165
No files matched your search
@@ -245,7 +245,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
|
||||
dataDevices = []osddaemon.DesiredDevice{
|
||||
{Name: osdDataDeviceFilter, IsFilter: true},
|
||||
{Name: osdDataDeviceFilter, IsFilter: true, OSDsPerDevice: cfg.storeConfig.OSDsPerDevice},
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
|
||||
@@ -186,7 +186,7 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi
|
||||
if device.Type == sys.PartType {
|
||||
continue
|
||||
}
|
||||
ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name)
|
||||
partCount, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err)
|
||||
}
|
||||
@@ -197,12 +197,13 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi
|
||||
continue
|
||||
}
|
||||
|
||||
var deviceInfo *DeviceOsdIDEntry
|
||||
if metadataDevice != "" && metadataDevice == device.Name {
|
||||
// current device is desired as the metadata device
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}, LegacyPartitionsFound: ownPartitions}
|
||||
deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID, Metadata: []int{}}
|
||||
} else if len(desiredDevices) == 1 && desiredDevices[0].Name == "all" {
|
||||
// user has specified all devices, use the current one for data
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, LegacyPartitionsFound: ownPartitions}
|
||||
deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID}
|
||||
} else if len(desiredDevices) > 0 {
|
||||
var matched bool
|
||||
var err error
|
||||
@@ -211,8 +212,13 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi
|
||||
if desiredDevice.IsFilter {
|
||||
// the desired devices is a regular expression
|
||||
matched, err = regexp.Match(desiredDevice.Name, []byte(device.Name))
|
||||
}
|
||||
if device.Name == desiredDevice.Name {
|
||||
if err != nil {
|
||||
logger.Errorf("regex failed on device %s and filter %s. %+v", device.Name, desiredDevice.Name, err)
|
||||
continue
|
||||
}
|
||||
logger.Infof("device %s matches device filter %s: %t", device.Name, desiredDevice.Name, matched)
|
||||
} else if device.Name == desiredDevice.Name {
|
||||
logger.Infof("%s found in the desired devices", device.Name)
|
||||
matched = true
|
||||
}
|
||||
matchedDevice = desiredDevice
|
||||
@@ -223,13 +229,20 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi
|
||||
|
||||
if err == nil && matched {
|
||||
// the current device matches the user specifies filter/list, use it for data
|
||||
available.Entries[device.Name] = &DeviceOsdIDEntry{Data: unassignedOSDID, Config: matchedDevice}
|
||||
deviceInfo = &DeviceOsdIDEntry{Data: unassignedOSDID, Config: matchedDevice}
|
||||
} else {
|
||||
logger.Infof("skipping device %s that does not match the device filter/list (%v). %+v", device.Name, desiredDevices, err)
|
||||
}
|
||||
} else {
|
||||
logger.Infof("skipping device %s until the admin specifies it can be used by an osd", device.Name)
|
||||
}
|
||||
|
||||
if deviceInfo != nil {
|
||||
if partCount > 0 {
|
||||
deviceInfo.LegacyPartitionsFound = ownPartitions
|
||||
}
|
||||
available.Entries[device.Name] = deviceInfo
|
||||
}
|
||||
}
|
||||
|
||||
return available, nil
|
||||
|
||||
@@ -110,7 +110,7 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool
|
||||
}
|
||||
|
||||
// check if partitions belong to rook
|
||||
ownPartitions, devFS, err := sys.CheckIfDeviceAvailable(context.Executor, dataDetails.Device)
|
||||
_, ownPartitions, devFS, err := sys.CheckIfDeviceAvailable(context.Executor, dataDetails.Device)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to format device. %+v", err)
|
||||
}
|
||||
@@ -170,7 +170,7 @@ func partitionMetadata(context *clusterd.Context, info *config.MetadataDeviceInf
|
||||
}
|
||||
|
||||
// check one last time to make sure it's OK for us to format this metadata device
|
||||
ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, info.Device)
|
||||
_, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, info.Device)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get metadata device %s info: %+v", info.Device, err)
|
||||
} else if fs != "" || !ownPartitions {
|
||||
|
||||
@@ -76,7 +76,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
|
||||
|
||||
batchArgs := append(baseArgs, []string{
|
||||
osdsPerDeviceFlag,
|
||||
strconv.Itoa(a.storeConfig.OSDsPerDevice),
|
||||
sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice),
|
||||
}...)
|
||||
|
||||
// ceph-volume is soon implementing a parameter to specify the "fast devices", which correspond to the "metadataDevice" from the
|
||||
@@ -102,7 +102,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
|
||||
immediateExecuteArgs := append(baseArgs, []string{
|
||||
deviceArg,
|
||||
osdsPerDeviceFlag,
|
||||
strconv.Itoa(device.Config.OSDsPerDevice),
|
||||
sanitizeOSDsPerDevice(device.Config.OSDsPerDevice),
|
||||
}...)
|
||||
|
||||
if err := context.Executor.ExecuteCommand(false, "", cephVolumeCmd, immediateExecuteArgs...); err != nil {
|
||||
@@ -123,6 +123,14 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeOSDsPerDevice(count int) string {
|
||||
if count < 1 {
|
||||
count = 1
|
||||
}
|
||||
return strconv.Itoa(count)
|
||||
}
|
||||
|
||||
func getCephVolumeSupported(context *clusterd.Context) (bool, error) {
|
||||
_, err := context.Executor.ExecuteCommandWithOutput(false, "", cephVolumeCmd, "lvm", "batch", "--prepare")
|
||||
if err != nil {
|
||||
|
||||
@@ -107,3 +107,10 @@ func TestParseCephVolumeResult(t *testing.T) {
|
||||
require.NotNil(t, osds)
|
||||
assert.Equal(t, 2, len(osds))
|
||||
}
|
||||
|
||||
func TestSanitizeOSDsPerDevice(t *testing.T) {
|
||||
assert.Equal(t, "1", sanitizeOSDsPerDevice(-1))
|
||||
assert.Equal(t, "1", sanitizeOSDsPerDevice(0))
|
||||
assert.Equal(t, "1", sanitizeOSDsPerDevice(1))
|
||||
assert.Equal(t, "2", sanitizeOSDsPerDevice(2))
|
||||
}
|
||||
@@ -37,7 +37,6 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object/user"
|
||||
"github.com/rook/rook/pkg/operator/ceph/pool"
|
||||
"github.com/rook/rook/pkg/operator/discover"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -412,7 +411,6 @@ func (c *ClusterController) onDelete(obj interface{}) {
|
||||
if clust.Spec.Storage.AnyUseAllDevices() {
|
||||
c.devicesInUse = false
|
||||
}
|
||||
discover.FreeDevicesByCluster(c.context, clust.Namespace)
|
||||
}
|
||||
|
||||
func (c *ClusterController) handleDelete(cluster *cephv1.CephCluster, retryInterval time.Duration) error {
|
||||
|
||||
@@ -174,13 +174,8 @@ func (c *Cluster) Start() error {
|
||||
logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes))
|
||||
c.Storage.Nodes = validNodes
|
||||
|
||||
// orchestrate individual nodes, starting with any that are still ongoing (in the case that we
|
||||
// are resuming a previous orchestration attempt)
|
||||
config := newProvisionConfig()
|
||||
logger.Infof("checking if orchestration is still in progress")
|
||||
c.completeProvisionSkipOSDStart(config)
|
||||
|
||||
// start the jobs to provision the OSD devices and directories
|
||||
config := newProvisionConfig()
|
||||
logger.Infof("start provisioning the osds on nodes, if needed")
|
||||
c.startProvisioning(config)
|
||||
|
||||
@@ -202,7 +197,10 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
|
||||
func (c *Cluster) startProvisioning(config *provisionConfig) {
|
||||
config.devicesToUse = make(map[string][]rookalpha.Device)
|
||||
if len(c.dataDirHostPath) == 0 {
|
||||
logger.Warningf("skipping osd provisioning where no dataDirHostPath is set")
|
||||
return
|
||||
}
|
||||
|
||||
// start with nodes currently in the storage spec
|
||||
for _, node := range c.Storage.Nodes {
|
||||
@@ -224,23 +222,11 @@ func (c *Cluster) startProvisioning(config *provisionConfig) {
|
||||
config.addError("failed to set orchestration starting status for node %s: %+v", n.Name, err)
|
||||
continue
|
||||
}
|
||||
config.devicesToUse[n.Name] = n.Devices
|
||||
availDev, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace, n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices())
|
||||
if deviceErr != nil {
|
||||
logger.Warningf("failed to get devices for node %s cluster %s: %v", n.Name, c.Namespace, deviceErr)
|
||||
} else {
|
||||
config.devicesToUse[n.Name] = availDev
|
||||
logger.Infof("avail devices for node %s: %+v", n.Name, availDev)
|
||||
}
|
||||
if len(availDev) == 0 && len(c.dataDirHostPath) == 0 {
|
||||
config.addError("empty volumes for node %s", n.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
// create the job that prepares osds on the node
|
||||
storeConfig := osdconfig.ToStoreConfig(n.Config)
|
||||
metadataDevice := osdconfig.MetadataDevice(n.Config)
|
||||
job, err := c.makeJob(n.Name, config.devicesToUse[n.Name], n.Selection, n.Resources, storeConfig, metadataDevice, n.Location)
|
||||
job, err := c.makeJob(n.Name, n.Devices, n.Selection, n.Resources, storeConfig, metadataDevice, n.Location)
|
||||
if err != nil {
|
||||
message := fmt.Sprintf("failed to create prepare job node %s: %v", n.Name, err)
|
||||
config.addError(message)
|
||||
@@ -252,8 +238,9 @@ func (c *Cluster) startProvisioning(config *provisionConfig) {
|
||||
}
|
||||
|
||||
if !c.runJob(job, n.Name, config, "provision") {
|
||||
if err = discover.FreeDevices(c.context, n.Name, c.Namespace); err != nil {
|
||||
logger.Warningf("failed to free devices: %s", err)
|
||||
status := OrchestrationStatus{Status: OrchestrationStatusCompleted, Message: fmt.Sprintf("failed to start osd provisioning on node %s", n.Name)}
|
||||
if err := c.updateNodeStatus(n.Name, status); err != nil {
|
||||
config.addError("failed to update node %s status. %+v", n.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -293,30 +280,18 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig
|
||||
// start osds
|
||||
for _, osd := range osds {
|
||||
logger.Debugf("start osd %v", osd)
|
||||
dp, err := c.makeDeployment(n.Name, config.devicesToUse[n.Name], n.Selection, n.Resources, storeConfig, metadataDevice, n.Location, osd)
|
||||
dp, err := c.makeDeployment(n.Name, n.Selection, n.Resources, storeConfig, metadataDevice, n.Location, osd)
|
||||
if err != nil {
|
||||
errMsg := fmt.Sprintf("nil deployment for node %s: %v", n.Name, err)
|
||||
errMsg := fmt.Sprintf("failed to create deployment for node %s: %v", n.Name, err)
|
||||
config.addError(errMsg)
|
||||
err = discover.FreeDevices(c.context, n.Name, c.Namespace)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to free devices: %s", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if err = c.deleteDeploymentWithLegacyName(osd.ID); err != nil {
|
||||
logger.Warningf("failed to delete legacy osd deployment. %+v", err)
|
||||
}
|
||||
|
||||
_, err = c.context.Clientset.Apps().Deployments(c.Namespace).Create(dp)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
// we failed to create job, update the orchestration status for this node
|
||||
logger.Warningf("failed to create osd deployment for node %s, osd %v: %+v", n.Name, osd, err)
|
||||
err = discover.FreeDevices(c.context, n.Name, c.Namespace)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to free devices: %s", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID)
|
||||
@@ -329,11 +304,6 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) deleteDeploymentWithLegacyName(osdID int) error {
|
||||
legacyName := fmt.Sprintf(legacyAppNameFmt, osdID)
|
||||
return k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, legacyName)
|
||||
}
|
||||
|
||||
func (c *Cluster) handleRemovedNodes(config *provisionConfig) {
|
||||
// find all removed nodes (if any) and start orchestration to remove them from the cluster
|
||||
removedNodes, err := c.findRemovedNodes()
|
||||
|
||||
@@ -88,35 +88,6 @@ func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake
|
||||
return err
|
||||
}
|
||||
|
||||
func TestLegacyDeployment(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "myversion", cephv1.CephVersionSpec{},
|
||||
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
osdID := 23
|
||||
d := &apps.Deployment{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf(legacyAppNameFmt, osdID), Namespace: c.Namespace}}
|
||||
_, err := clientset.Apps().Deployments(c.Namespace).Create(d)
|
||||
require.Nil(t, err)
|
||||
|
||||
// delete the deployment
|
||||
assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID))
|
||||
deployments, err := clientset.Apps().Deployments(c.Namespace).List(metav1.ListOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(deployments.Items))
|
||||
|
||||
// return success if the deployment doesn't exist
|
||||
assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID))
|
||||
|
||||
// don't delete the newer deployment name
|
||||
d = &apps.Deployment{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf(osdAppNameFmt, osdID), Namespace: c.Namespace}}
|
||||
_, err = clientset.Apps().Deployments(c.Namespace).Create(d)
|
||||
require.Nil(t, err)
|
||||
assert.Nil(t, c.deleteDeploymentWithLegacyName(osdID))
|
||||
deployments, err = clientset.Apps().Deployments(c.Namespace).List(metav1.ListOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(deployments.Items))
|
||||
}
|
||||
|
||||
func TestAddRemoveNode(t *testing.T) {
|
||||
// create a storage spec with the given nodes/devices/dirs
|
||||
nodeName := "node8230"
|
||||
@@ -146,7 +117,7 @@ func TestAddRemoveNode(t *testing.T) {
|
||||
clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(statusMapWatcher, nil))
|
||||
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1.CephVersionSpec{},
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
storageSpec, "/foo", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// kick off the start of the orchestration in a goroutine
|
||||
var startErr error
|
||||
@@ -275,17 +246,17 @@ func TestDiscoverOSDs(t *testing.T) {
|
||||
node2 := "n2"
|
||||
|
||||
osd1 := OSDInfo{ID: 0, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"}
|
||||
d1, err := c.makeDeployment(node1, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd1)
|
||||
d1, err := c.makeDeployment(node1, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd1)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, d1)
|
||||
|
||||
osd2 := OSDInfo{ID: 101, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"}
|
||||
d2, err := c.makeDeployment(node1, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd2)
|
||||
d2, err := c.makeDeployment(node1, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd2)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, d2)
|
||||
|
||||
osd3 := OSDInfo{ID: 23, IsDirectory: true, IsFileStore: true, DataPath: "/rook/path"}
|
||||
d3, err := c.makeDeployment(node2, []rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd3)
|
||||
d3, err := c.makeDeployment(node2, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", osd3)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, d3)
|
||||
|
||||
@@ -338,7 +309,7 @@ func TestAddNodeFailure(t *testing.T) {
|
||||
assert.Nil(t, cmErr)
|
||||
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", cephv1.CephVersionSpec{},
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
storageSpec, "/foo", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// kick off the start of the orchestration in a goroutine
|
||||
var startErr error
|
||||
|
||||
@@ -76,7 +76,7 @@ func (c *Cluster) makeJob(nodeName string, devices []rookalpha.Device,
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements,
|
||||
func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, resources v1.ResourceRequirements,
|
||||
storeConfig config.StoreConfig, metadataDevice, location string, osd OSDInfo) (*apps.Deployment, error) {
|
||||
|
||||
replicaCount := int32(1)
|
||||
|
||||
@@ -86,7 +86,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
|
||||
ID: 0,
|
||||
}
|
||||
|
||||
deployment, err := c.makeDeployment(n.Name, devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd)
|
||||
deployment, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, deployment)
|
||||
assert.Equal(t, "rook-ceph-osd-0", deployment.Name)
|
||||
@@ -165,7 +165,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
|
||||
IsDirectory: true,
|
||||
DataPath: "/my/root/path/osd1",
|
||||
}
|
||||
deployment, err := c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd)
|
||||
deployment, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd)
|
||||
assert.NotNil(t, deployment)
|
||||
assert.Nil(t, err)
|
||||
// pod spec should have a volume for the given dir in the main container and the init container
|
||||
@@ -192,7 +192,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
|
||||
IsDirectory: true,
|
||||
DataPath: "/var/lib/rook/osd1",
|
||||
}
|
||||
deployment, err = c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd)
|
||||
deployment, err = c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd)
|
||||
assert.NotNil(t, deployment)
|
||||
assert.Nil(t, err)
|
||||
// pod spec should have a volume for the given dir in the main container and the init container
|
||||
@@ -300,7 +300,7 @@ func TestHostNetwork(t *testing.T) {
|
||||
osd := OSDInfo{
|
||||
ID: 0,
|
||||
}
|
||||
r, err := c.makeDeployment(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd)
|
||||
r, err := c.makeDeployment(n.Name, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location, osd)
|
||||
assert.NotNil(t, r)
|
||||
assert.Nil(t, err)
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
@@ -48,7 +47,6 @@ const (
|
||||
)
|
||||
|
||||
type provisionConfig struct {
|
||||
devicesToUse map[string][]rookalpha.Device
|
||||
errorMessages []string
|
||||
}
|
||||
|
||||
|
||||
@@ -298,44 +298,6 @@ func ListDevicesInUse(context *clusterd.Context, namespace, nodeName string) ([]
|
||||
return devices, nil
|
||||
}
|
||||
|
||||
// FreeDevices frees up devices used by a cluster on a node.
|
||||
func FreeDevices(context *clusterd.Context, nodeName, clusterName string) error {
|
||||
if len(nodeName) == 0 || len(clusterName) == 0 {
|
||||
return nil
|
||||
}
|
||||
namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
cmName := k8sutil.TruncateNodeName(fmt.Sprintf(deviceInUseCMName, clusterName, "%s"), nodeName)
|
||||
// delete configmap
|
||||
err := context.Clientset.CoreV1().ConfigMaps(namespace).Delete(cmName, &metav1.DeleteOptions{})
|
||||
if err != nil && !kserrors.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to delete configmap %s/%s. %+v", namespace, cmName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FreeDevicesByCluster frees devices on all nodes that are used by the cluster
|
||||
func FreeDevicesByCluster(context *clusterd.Context, clusterName string) error {
|
||||
logger.Infof("freeing devices used by cluster %s", clusterName)
|
||||
namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", deviceInUseClusterAttr, clusterName)}
|
||||
cms, err := context.Clientset.CoreV1().ConfigMaps(namespace).List(listOpts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list device in use configmaps for cluster %s: %+v", clusterName, err)
|
||||
}
|
||||
|
||||
for _, cm := range cms.Items {
|
||||
// delete configmap
|
||||
cmName := cm.Name
|
||||
err := context.Clientset.CoreV1().ConfigMaps(namespace).Delete(cmName, &metav1.DeleteOptions{})
|
||||
logger.Infof("deleting configmap %s: %+v", cmName, err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAvailableDevices conducts outer join using input filters with free devices that a node has. It marks the devices from join result as in-use.
|
||||
func GetAvailableDevices(context *clusterd.Context, nodeName, clusterName string, devices []rookalpha.Device, filter string, useAllDevices bool) ([]rookalpha.Device, error) {
|
||||
results := []rookalpha.Device{}
|
||||
|
||||
@@ -116,29 +116,4 @@ func TestGetAvailableDevices(t *testing.T) {
|
||||
devices, err = GetAvailableDevices(context, nodeName, ns, d, "^sd.", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(devices))
|
||||
|
||||
err = FreeDevices(context, nodeName, ns)
|
||||
assert.Nil(t, err)
|
||||
// all devices freed
|
||||
devices, err = GetAvailableDevices(context, nodeName, ns, nil, "^sd.", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 4, len(devices))
|
||||
// devices should be in use now, 2nd try gets the same list
|
||||
devices, err = GetAvailableDevices(context, nodeName, ns, nil, "^sd.", false)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 4, len(devices))
|
||||
|
||||
err = FreeDevices(context, nodeName, ns)
|
||||
assert.Nil(t, err)
|
||||
|
||||
devices, err = GetAvailableDevices(context, nodeName, ns, nil, "", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 5, len(devices))
|
||||
// devices should be in use now, 2nd try gets the same list
|
||||
devices, err = GetAvailableDevices(context, nodeName, ns, nil, "", true)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 5, len(devices))
|
||||
|
||||
err = FreeDevices(context, nodeName, ns)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
opkit "github.com/rook/operator-kit"
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/discover"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/iscsi"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/isgw"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/nfs"
|
||||
@@ -322,7 +321,6 @@ func (c *ClusterController) onDelete(obj interface{}) {
|
||||
if clust.Spec.Storage.AnyUseAllDevices() {
|
||||
c.devicesInUse = false
|
||||
}
|
||||
discover.FreeDevicesByCluster(c.context, clust.Namespace)
|
||||
}
|
||||
|
||||
func (c *ClusterController) handleDelete(clust *edgefsv1alpha1.Cluster, retryInterval time.Duration) error {
|
||||
|
||||
+10
-4
@@ -289,12 +289,18 @@ func UnmountDevice(devicePath string, executor exec.Executor) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func CheckIfDeviceAvailable(executor exec.Executor, name string) (bool, string, error) {
|
||||
// CheckIfDeviceAvailable checks if a device is available for consumption. The caller
|
||||
// needs to decide based on the return values whether it is available. The return values are
|
||||
// the number of partitions, whether Rook has created partitions on the device in the past
|
||||
// possibly from the same or a previous cluster, the filesystem found, or an err if failed
|
||||
// to retrieve the properties.
|
||||
func CheckIfDeviceAvailable(executor exec.Executor, name string) (int, bool, string, error) {
|
||||
ownPartitions := true
|
||||
partitions, _, err := GetDevicePartitions(name, executor)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err)
|
||||
return 0, false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err)
|
||||
}
|
||||
partCount := len(partitions)
|
||||
if !RookOwnsPartitions(partitions) {
|
||||
ownPartitions = false
|
||||
}
|
||||
@@ -302,10 +308,10 @@ func CheckIfDeviceAvailable(executor exec.Executor, name string) (bool, string,
|
||||
// check if there is a file system on the device
|
||||
devFS, err := GetDeviceFilesystems(name, executor)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err)
|
||||
return 0, false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err)
|
||||
}
|
||||
|
||||
return ownPartitions, devFS, nil
|
||||
return partCount, ownPartitions, devFS, nil
|
||||
}
|
||||
|
||||
// RookOwnsPartitions check if all partitions in list are owned by Rook
|
||||
|
||||
@@ -40,7 +40,7 @@ func IsAdditionalDeviceAvailableOnCluster() bool {
|
||||
continue
|
||||
}
|
||||
|
||||
ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device)
|
||||
_, ownPartitions, fs, err := sys.CheckIfDeviceAvailable(executor, device)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to detect device %s availability. %+v", device, err)
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user