From 322549cbe1e332e60cea55396d77af19996c851b Mon Sep 17 00:00:00 2001 From: sp98 Date: Mon, 13 May 2024 15:33:41 +0530 Subject: [PATCH] osd: skip upgrade if migration is pending Currently we allow upgrade of other OSDs while migration of OSD, due to change in backing store, is pending. This will not work if the updated OSD ceph version does not support the currently applied backing store. So rook will skip any OSD upgrade if the OSD migration is pending. Signed-off-by: sp98 --- pkg/operator/ceph/cluster/osd/osd.go | 68 ++++------ pkg/operator/ceph/cluster/osd/osd_test.go | 17 +-- pkg/operator/ceph/cluster/osd/replace.go | 76 +++++++---- pkg/operator/ceph/cluster/osd/replace_test.go | 125 ------------------ pkg/operator/ceph/cluster/osd/update.go | 22 ++- 5 files changed, 101 insertions(+), 207 deletions(-) diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 5922e15c3..cab46aa5c 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -216,10 +216,10 @@ func (c *Cluster) Start() error { } logger.Infof("wait timeout for healthy OSDs during upgrade or restart is %q", c.clusterInfo.OsdUpgradeTimeout) - // identify OSDs that need migration to a new backend store - err := c.replaceOSDForNewStore() + // replace OSDs for a new backing store + osdsToBeReplaced, err := c.replaceOSDForNewStore() if err != nil { - return errors.Wrapf(err, "failed to replace an OSD that needs migration to new backend store in namespace %q", namespace) + return errors.Wrapf(err, "failed to replace OSD for new backing store %q in namespace %q", c.spec.Storage.Store.Type, namespace) } osdsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(c.clusterInfo.Context, c.context, c.clusterInfo.Namespace, OsdIdLabelKey, AppName) @@ -233,6 +233,11 @@ func (c *Cluster) Start() error { return errors.Wrapf(err, "failed to get information about currently-running OSD Deployments in namespace %q", namespace) } + // OSDs that are to be replaced should not be upgraded. So remove them from the `updateQueue` + if len(osdsToBeReplaced) > 0 { + updateQueue.Remove(osdsToBeReplaced.getOSDIds()) + } + logger.Debugf("%d of %d OSD Deployments need update", updateQueue.Len(), deployments.Len()) updateConfig := c.newUpdateConfig(config, updateQueue, deployments, osdsToSkipReconcile) @@ -284,27 +289,23 @@ func (c *Cluster) Start() error { } if c.spec.Storage.Store.UpdateStore == OSDStoreUpdateConfirmation { - if c.replaceOSD != nil { - delOpts := &k8sutil.DeleteOptions{MustDelete: true, WaitOptions: k8sutil.WaitOptions{Wait: true}} - err := k8sutil.DeleteConfigMap(c.clusterInfo.Context, c.context.Clientset, OSDReplaceConfigName, namespace, delOpts) - if err != nil { + delOpts := &k8sutil.DeleteOptions{MustDelete: true, WaitOptions: k8sutil.WaitOptions{Wait: true}} + err := k8sutil.DeleteConfigMap(c.clusterInfo.Context, c.context.Clientset, OSDReplaceConfigName, namespace, delOpts) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Debugf("config map %q not found. Ignoring since object must be deleted.", OSDReplaceConfigName) + } else { return errors.Wrapf(err, "failed to delete the %q configmap", OSDReplaceConfigName) } } - // wait for the pgs to be healthy before attempting to migrate the next OSD - _, err := c.waitForHealthyPGs() - if err != nil { - return errors.Wrapf(err, "failed to wait for pgs to be healhty") - } - - // reconcile if migration of one or more OSD is pending. - osdsToReplace, err := c.getOSDWithNonMatchingStore() - if err != nil { - return errors.Wrapf(err, "failed to check if any OSD migration is pending") - } - - if len(osdsToReplace) != 0 { + if len(osdsToBeReplaced) > 0 { + // wait for the pgs to be healthy before attempting to migrate the next OSD + _, err := c.waitForHealthyPGs() + if err != nil { + return errors.Wrapf(err, "failed to wait for pgs to be healhty") + } + // return with error to reconcile the operator since there are OSDs that are pending migration return errors.New("reconcile operator to replace OSDs that are pending migration") } } @@ -825,27 +826,10 @@ func (c *Cluster) applyUpgradeOSDFunctionality() { } } -// replaceOSDForNewStore deletes an existing OSD deployment that does not match the expected OSD backend store provided in the storage spec -func (c *Cluster) replaceOSDForNewStore() error { - if c.spec.Storage.Store.UpdateStore != OSDStoreUpdateConfirmation { - logger.Debugf("no OSD migration to a new backend store is requested") - return nil - } - - osdToReplace, err := c.getOSDReplaceInfo() - if err != nil { - return errors.Wrapf(err, "failed to get OSD replace info") - } - - if osdToReplace == nil { - logger.Info("no osd to replace") - return nil - } - - logger.Infof("starting migration of the OSD.%d", osdToReplace.ID) - +// deleteOSDDeployment deletes an existing OSD deployment and saves the information in the configmap +func (c *Cluster) deleteOSDDeployment(osdID int) error { // Delete the OSD deployment - deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdToReplace.ID) + deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdID) logger.Infof("removing the OSD deployment %q", deploymentName) if err := k8sutil.DeleteDeployment(c.clusterInfo.Context, c.context.Clientset, c.clusterInfo.Namespace, deploymentName); err != nil { if err != nil { @@ -857,9 +841,7 @@ func (c *Cluster) replaceOSDForNewStore() error { } } - c.replaceOSD = osdToReplace - - return osdToReplace.saveAsConfig(c.context, c.clusterInfo) + return c.replaceOSD.saveAsConfig(c.context, c.clusterInfo) } func (c *Cluster) waitForHealthyPGs() (bool, error) { diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index 043e41c7f..062fe3614 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -19,7 +19,6 @@ package osd import ( "context" "encoding/json" - "fmt" "testing" "github.com/pkg/errors" @@ -728,7 +727,7 @@ func TestReplaceOSDForNewStore(t *testing.T) { }, } c := New(context, clusterInfo, spec, "myversion") - err := c.replaceOSDForNewStore() + _, err := c.replaceOSDForNewStore() assert.NoError(t, err) assert.Nil(t, c.replaceOSD) }) @@ -743,7 +742,7 @@ func TestReplaceOSDForNewStore(t *testing.T) { }, } c := New(context, clusterInfo, spec, "myversion") - err := c.replaceOSDForNewStore() + _, err := c.replaceOSDForNewStore() assert.NoError(t, err) assert.Nil(t, c.replaceOSD) }) @@ -761,8 +760,9 @@ func TestReplaceOSDForNewStore(t *testing.T) { d := getDummyDeploymentOnNode(clientset, c, "node2", 0) d.Labels[osdStore] = "bluestore-rdr" createDeploymentOrPanic(clientset, d) - err := c.replaceOSDForNewStore() + osdsToBeReplaced, err := c.replaceOSDForNewStore() assert.NoError(t, err) + assert.Equal(t, 0, len(osdsToBeReplaced)) assert.Nil(t, c.replaceOSD) }) @@ -779,7 +779,7 @@ func TestReplaceOSDForNewStore(t *testing.T) { // create osd deployment with `bluestore` backend store d := getDummyDeploymentOnNode(clientset, c, "node2", 1) createDeploymentOrPanic(clientset, d) - err := c.replaceOSDForNewStore() + _, err := c.replaceOSDForNewStore() assert.NoError(t, err) assert.NotNil(t, c.replaceOSD) assert.Equal(t, 1, c.replaceOSD.ID) @@ -816,9 +816,9 @@ func TestReplaceOSDForNewStore(t *testing.T) { c := New(context, clusterInfo, spec, "myversion") d := getDummyDeploymentOnPVC(clientset, c, "pvc1", 2) createDeploymentOrPanic(clientset, d) - err := c.replaceOSDForNewStore() + osdsToBeReplaced, err := c.replaceOSDForNewStore() assert.NoError(t, err) - fmt.Printf("%+v", c.replaceOSD) + assert.Equal(t, 1, len(osdsToBeReplaced)) assert.NotNil(t, c.replaceOSD) assert.Equal(t, 2, c.replaceOSD.ID) assert.Equal(t, "pvc1", c.replaceOSD.Path) @@ -862,8 +862,9 @@ func TestReplaceOSDForNewStore(t *testing.T) { } context.Executor = executor c := New(context, clusterInfo, spec, "myversion") - err := c.replaceOSDForNewStore() + osdsToBeReplaced, err := c.replaceOSDForNewStore() assert.NoError(t, err) + assert.Equal(t, 0, len(osdsToBeReplaced)) assert.Nil(t, c.replaceOSD) }) } diff --git a/pkg/operator/ceph/cluster/osd/replace.go b/pkg/operator/ceph/cluster/osd/replace.go index 2c51029f6..845ad7249 100644 --- a/pkg/operator/ceph/cluster/osd/replace.go +++ b/pkg/operator/ceph/cluster/osd/replace.go @@ -19,7 +19,6 @@ package osd import ( "encoding/json" - "fmt" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" @@ -43,6 +42,16 @@ type OSDReplaceInfo struct { Node string `json:"node"` } +type OSDReplaceInfoList []OSDReplaceInfo + +func (o *OSDReplaceInfoList) getOSDIds() []int { + osdIDs := []int{} + for _, osd := range *o { + osdIDs = append(osdIDs, osd.ID) + } + return osdIDs +} + func (o *OSDReplaceInfo) saveAsConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error { configStr, err := o.string() if err != nil { @@ -81,58 +90,67 @@ func (o *OSDReplaceInfo) string() (string, error) { return string(configInBytes), nil } -// getOSDReplaceInfo returns an existing OSD that needs to be replaced for a new backend store -func (c *Cluster) getOSDReplaceInfo() (*OSDReplaceInfo, error) { - osdReplaceInfo, err := GetOSDReplaceConfigMap(c.context, c.clusterInfo) +func (c *Cluster) replaceOSDForNewStore() (OSDReplaceInfoList, error) { + if c.spec.Storage.Store.UpdateStore != OSDStoreUpdateConfirmation { + logger.Info("no replacement of osds is requested") + return nil, nil + } + + osdsToBeReplaced, err := c.getOSDWithNonMatchingStore() if err != nil { - return nil, errors.Wrap(err, "failed to get any existing OSD in replace configmap") + return nil, errors.Wrapf(err, "failed to get information about the OSDs where backing store does not match the spec in namespace %q", c.clusterInfo.Namespace) } - if osdReplaceInfo != nil { - return osdReplaceInfo, nil + if len(osdsToBeReplaced) == 0 { + logger.Debug("all OSDs are using the desired backing store. No replacement is required.") + return osdsToBeReplaced, nil } + // replace an OSD only if Pgs are healthy pgHealthMsg, pgClean, err := cephclient.IsClusterClean(c.context, c.clusterInfo, c.spec.DisruptionManagement.PGHealthyRegex) if err != nil { return nil, errors.Wrapf(err, "failed to check if the pgs are clean before replacing OSDs") } + if !pgClean { logger.Warningf("skipping OSD replacement because pgs are not healthy. PG status: %q", pgHealthMsg) - return nil, nil + return osdsToBeReplaced, nil } - logger.Infof("placement group status: %q", pgHealthMsg) - - osdsToReplace, err := c.getOSDWithNonMatchingStore() + // Check for an existing OSDs in the configmap + osdToBeReplaced, err := GetOSDReplaceConfigMap(c.context, c.clusterInfo) if err != nil { - return nil, errors.Wrap(err, "failed to list out OSDs with non matching backend store") + return nil, errors.Wrap(err, "failed to get any existing OSD in replace configmap") + } + if osdToBeReplaced != nil { + c.replaceOSD = osdToBeReplaced + } else { + c.replaceOSD = &osdsToBeReplaced[0] } - if len(osdsToReplace) == 0 { - logger.Infof("all osds have already been migrated to backend store %q", c.spec.Storage.Store.Type) - return nil, nil + logger.Infof("replacing OSD.%d to the new backing store %q", c.replaceOSD.ID, c.spec.Storage.Store.Type) + err = c.deleteOSDDeployment(c.replaceOSD.ID) + if err != nil { + return nil, errors.Wrapf(err, "failed to delete OSD deployment that needs migration to new backend store in namespace %q", c.clusterInfo.Namespace) } - logger.Infof("%d osd(s) require migration to new backend store %q.", len(osdsToReplace), c.spec.Storage.Store.Type) - - return &osdsToReplace[0], nil + return osdsToBeReplaced, nil } // getOSDWithNonMatchingStore returns OSDs with osd-store label different from expected store in cephCluster spec -func (c *Cluster) getOSDWithNonMatchingStore() ([]OSDReplaceInfo, error) { - listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)} - deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts) - if err != nil { - return nil, errors.Wrap(err, "failed to query OSDs to skip reconcile") - } - +func (c *Cluster) getOSDWithNonMatchingStore() (OSDReplaceInfoList, error) { osdReplaceList := []OSDReplaceInfo{} - for i := range deployments.Items { - if osdStore, ok := deployments.Items[i].Labels[osdStore]; ok { + // get existing OSD deployments + osdDeployments, err := c.getOSDDeployments() + if err != nil { + return osdReplaceList, errors.Wrapf(err, "failed to get existing OSD deployments in namespace %q", c.clusterInfo.Namespace) + } + for i := range osdDeployments.Items { + if osdStore, ok := osdDeployments.Items[i].Labels[osdStore]; ok { if osdStore != string(c.spec.Storage.Store.Type) { - osdInfo, err := c.getOSDInfo(&deployments.Items[i]) + osdInfo, err := c.getOSDInfo(&osdDeployments.Items[i]) if err != nil { - return nil, errors.Wrapf(err, "failed to details about the OSD %q", deployments.Items[i].Name) + return nil, errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name) } var path string if osdInfo.PVCName != "" { diff --git a/pkg/operator/ceph/cluster/osd/replace_test.go b/pkg/operator/ceph/cluster/osd/replace_test.go index b2daf5cef..e22fc8ae0 100644 --- a/pkg/operator/ceph/cluster/osd/replace_test.go +++ b/pkg/operator/ceph/cluster/osd/replace_test.go @@ -21,12 +21,9 @@ import ( "context" "testing" - "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/operator/k8sutil" - exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" "k8s.io/client-go/kubernetes/fake" @@ -163,125 +160,3 @@ func TestGetOSDWithNonMatchingStoreOnPVCs(t *testing.T) { assert.Equal(t, "pvc0", osdList[0].Path) }) } - -func TestGetOSDReplaceInfo(t *testing.T) { - namespace := "rook-ceph" - clientset := fake.NewSimpleClientset() - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) - if args[0] == "status" { - return healthyCephStatus, nil - } - return "", errors.Errorf("unexpected ceph command '%v'", args) - }, - } - ctx := &clusterd.Context{ - Clientset: clientset, - Executor: executor, - } - clusterInfo := &cephclient.ClusterInfo{ - Namespace: namespace, - Context: context.TODO(), - } - clusterInfo.SetName("mycluster") - clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) - - spec := cephv1.ClusterSpec{} - c := New(ctx, clusterInfo, spec, "rook/rook:master") - - t.Run("no OSD replace info available", func(t *testing.T) { - actualOSDInfo, err := c.getOSDReplaceInfo() - assert.NoError(t, err) - assert.Nil(t, actualOSDInfo) - }) - - t.Run("ensure no OSD to replace if pgs are not healthy", func(t *testing.T) { - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) - if args[0] == "status" { - return unHealthyCephStatus, nil - } - return "", errors.Errorf("unexpected ceph command '%v'", args) - }, - } - - ctx.Executor = executor - c := New(ctx, clusterInfo, spec, "rook/rook:master") - actualOSDInfo, err := c.getOSDReplaceInfo() - assert.NoError(t, err) - assert.Nil(t, actualOSDInfo) - }) - - t.Run("read OSD 0 replace info from config map", func(t *testing.T) { - actualOSDInfo := &OSDReplaceInfo{ID: 0, Path: "/dev/sda", Node: "node1"} - err := actualOSDInfo.saveAsConfig(c.context, c.clusterInfo) - assert.NoError(t, err) - expectedOSDInfo, err := c.getOSDReplaceInfo() - assert.NoError(t, err) - assert.NotNil(t, expectedOSDInfo) - assert.Equal(t, 0, expectedOSDInfo.ID) - err = k8sutil.DeleteConfigMap(clusterInfo.Context, ctx.Clientset, OSDReplaceConfigName, namespace, &k8sutil.DeleteOptions{}) - assert.NoError(t, err) - }) - - t.Run("ensure no OSD replace info if all OSDs using expected OSD store", func(t *testing.T) { - spec := cephv1.ClusterSpec{ - Storage: cephv1.StorageScopeSpec{ - Store: cephv1.OSDStore{ - Type: "bluestore-rdr", - UpdateStore: "yes-really-update-store", - }, - }, - } - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) - if args[0] == "status" { - return healthyCephStatus, nil - } - return "", errors.Errorf("unexpected ceph command '%v'", args) - }, - } - - ctx.Executor = executor - c := New(ctx, clusterInfo, spec, "myversion") - d := getDummyDeploymentOnNode(clientset, c, "node2", 1) - d.Labels[osdStore] = "bluestore-rdr" - createDeploymentOrPanic(clientset, d) - expectedOSDInfo, err := c.getOSDReplaceInfo() - assert.NoError(t, err) - assert.Nil(t, expectedOSDInfo) - }) - - t.Run("ensure OSD replace info if OSD store is not up to date", func(t *testing.T) { - spec := cephv1.ClusterSpec{ - Storage: cephv1.StorageScopeSpec{ - Store: cephv1.OSDStore{ - Type: "bluestore-rdr", - UpdateStore: "yes-really-update-store", - }, - }, - } - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) - if args[0] == "status" { - return healthyCephStatus, nil - } - return "", errors.Errorf("unexpected ceph command '%v'", args) - }, - } - - ctx.Executor = executor - c := New(ctx, clusterInfo, spec, "myversion") - d := getDummyDeploymentOnNode(clientset, c, "node3", 2) - createDeploymentOrPanic(clientset, d) - expectedOSDInfo, err := c.getOSDReplaceInfo() - assert.NoError(t, err) - assert.NotNil(t, expectedOSDInfo) - assert.Equal(t, 2, expectedOSDInfo.ID) - assert.Equal(t, "node3", expectedOSDInfo.Node) - }) -} diff --git a/pkg/operator/ceph/cluster/osd/update.go b/pkg/operator/ceph/cluster/osd/update.go index 2bfcbb87a..ea4076fbe 100644 --- a/pkg/operator/ceph/cluster/osd/update.go +++ b/pkg/operator/ceph/cluster/osd/update.go @@ -232,7 +232,6 @@ func (c *Cluster) getOSDUpdateInfo(errs *provisionErrors) (*updateQueue, *existe updateQueue := newUpdateQueueWithCapacity(len(deps.Items)) existenceList := newExistenceListWithCapacity(len(deps.Items)) - for i := range deps.Items { id, err := getOSDID(&deps.Items[i]) // avoid implicit memory aliasing by indexing if err != nil { @@ -240,7 +239,6 @@ func (c *Cluster) getOSDUpdateInfo(errs *provisionErrors) (*updateQueue, *existe errs.addError("%v. did a user create their own deployment with label %q?", selector, err) continue } - // all OSD deployments should be marked as existing existenceList.Add(id) updateQueue.Push(id) @@ -299,6 +297,11 @@ func (q *updateQueue) Exists(osdID int) bool { return false } +// Clear deletes all entries inside the queue +func (q *updateQueue) Clear() { + q.q = q.q[:0] +} + // Remove removes the items from the queue if they exist. func (q *updateQueue) Remove(osdIDs []int) { shouldRemove := func(rid int) bool { @@ -369,3 +372,18 @@ func (c *Cluster) getFuncToListDeploymentsWithIDs(osdIDs []string) func() (*apps return c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts) } } + +// getOSDDeployments returns the list of existing OSD deployments +func (c *Cluster) getOSDDeployments() (*appsv1.DeploymentList, error) { + namespace := c.clusterInfo.Namespace + selector := fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName) + listOpts := metav1.ListOptions{ + // list only rook-ceph-osd Deployments + LabelSelector: selector, + } + deps, err := c.context.Clientset.AppsV1().Deployments(namespace).List(c.clusterInfo.Context, listOpts) + if err != nil { + return nil, errors.Wrap(err, "failed to query existing OSD deployments to check if they need to be updated") + } + return deps, nil +}