Merge pull request #14196 from sp98/skip-osd-uprade-during-migration

osd: skip upgrade if migration is pending
This commit is contained in:
Santosh Pillai authored and GitHub committed 2024-05-29 12:33:39 +05:30
commit f5ca09e385
5 files changed
+101 -207

No files matched your search

+25 -43
View File
@@ -216,10 +216,10 @@ func (c *Cluster) Start() error {
}
logger.Infof("wait timeout for healthy OSDs during upgrade or restart is %q", c.clusterInfo.OsdUpgradeTimeout)
// identify OSDs that need migration to a new backend store
err := c.replaceOSDForNewStore()
// replace OSDs for a new backing store
osdsToBeReplaced, err := c.replaceOSDForNewStore()
if err != nil {
return errors.Wrapf(err, "failed to replace an OSD that needs migration to new backend store in namespace %q", namespace)
return errors.Wrapf(err, "failed to replace OSD for new backing store %q in namespace %q", c.spec.Storage.Store.Type, namespace)
}
osdsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(c.clusterInfo.Context, c.context, c.clusterInfo.Namespace, OsdIdLabelKey, AppName)
@@ -233,6 +233,11 @@ func (c *Cluster) Start() error {
return errors.Wrapf(err, "failed to get information about currently-running OSD Deployments in namespace %q", namespace)
}
// OSDs that are to be replaced should not be upgraded. So remove them from the `updateQueue`
if len(osdsToBeReplaced) > 0 {
updateQueue.Remove(osdsToBeReplaced.getOSDIds())
}
logger.Debugf("%d of %d OSD Deployments need update", updateQueue.Len(), deployments.Len())
updateConfig := c.newUpdateConfig(config, updateQueue, deployments, osdsToSkipReconcile)
@@ -284,27 +289,23 @@ func (c *Cluster) Start() error {
}
if c.spec.Storage.Store.UpdateStore == OSDStoreUpdateConfirmation {
if c.replaceOSD != nil {
delOpts := &k8sutil.DeleteOptions{MustDelete: true, WaitOptions: k8sutil.WaitOptions{Wait: true}}
err := k8sutil.DeleteConfigMap(c.clusterInfo.Context, c.context.Clientset, OSDReplaceConfigName, namespace, delOpts)
if err != nil {
delOpts := &k8sutil.DeleteOptions{MustDelete: true, WaitOptions: k8sutil.WaitOptions{Wait: true}}
err := k8sutil.DeleteConfigMap(c.clusterInfo.Context, c.context.Clientset, OSDReplaceConfigName, namespace, delOpts)
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debugf("config map %q not found. Ignoring since object must be deleted.", OSDReplaceConfigName)
} else {
return errors.Wrapf(err, "failed to delete the %q configmap", OSDReplaceConfigName)
}
}
// wait for the pgs to be healthy before attempting to migrate the next OSD
_, err := c.waitForHealthyPGs()
if err != nil {
return errors.Wrapf(err, "failed to wait for pgs to be healhty")
}
// reconcile if migration of one or more OSD is pending.
osdsToReplace, err := c.getOSDWithNonMatchingStore()
if err != nil {
return errors.Wrapf(err, "failed to check if any OSD migration is pending")
}
if len(osdsToReplace) != 0 {
if len(osdsToBeReplaced) > 0 {
// wait for the pgs to be healthy before attempting to migrate the next OSD
_, err := c.waitForHealthyPGs()
if err != nil {
return errors.Wrapf(err, "failed to wait for pgs to be healhty")
}
// return with error to reconcile the operator since there are OSDs that are pending migration
return errors.New("reconcile operator to replace OSDs that are pending migration")
}
}
@@ -825,27 +826,10 @@ func (c *Cluster) applyUpgradeOSDFunctionality() {
}
}
// replaceOSDForNewStore deletes an existing OSD deployment that does not match the expected OSD backend store provided in the storage spec
func (c *Cluster) replaceOSDForNewStore() error {
if c.spec.Storage.Store.UpdateStore != OSDStoreUpdateConfirmation {
logger.Debugf("no OSD migration to a new backend store is requested")
return nil
}
osdToReplace, err := c.getOSDReplaceInfo()
if err != nil {
return errors.Wrapf(err, "failed to get OSD replace info")
}
if osdToReplace == nil {
logger.Info("no osd to replace")
return nil
}
logger.Infof("starting migration of the OSD.%d", osdToReplace.ID)
// deleteOSDDeployment deletes an existing OSD deployment and saves the information in the configmap
func (c *Cluster) deleteOSDDeployment(osdID int) error {
// Delete the OSD deployment
deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdToReplace.ID)
deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdID)
logger.Infof("removing the OSD deployment %q", deploymentName)
if err := k8sutil.DeleteDeployment(c.clusterInfo.Context, c.context.Clientset, c.clusterInfo.Namespace, deploymentName); err != nil {
if err != nil {
@@ -857,9 +841,7 @@ func (c *Cluster) replaceOSDForNewStore() error {
}
}
c.replaceOSD = osdToReplace
return osdToReplace.saveAsConfig(c.context, c.clusterInfo)
return c.replaceOSD.saveAsConfig(c.context, c.clusterInfo)
}
func (c *Cluster) waitForHealthyPGs() (bool, error) {
+9 -8
View File
@@ -19,7 +19,6 @@ package osd
import (
"context"
"encoding/json"
"fmt"
"testing"
"github.com/pkg/errors"
@@ -728,7 +727,7 @@ func TestReplaceOSDForNewStore(t *testing.T) {
},
}
c := New(context, clusterInfo, spec, "myversion")
err := c.replaceOSDForNewStore()
_, err := c.replaceOSDForNewStore()
assert.NoError(t, err)
assert.Nil(t, c.replaceOSD)
})
@@ -743,7 +742,7 @@ func TestReplaceOSDForNewStore(t *testing.T) {
},
}
c := New(context, clusterInfo, spec, "myversion")
err := c.replaceOSDForNewStore()
_, err := c.replaceOSDForNewStore()
assert.NoError(t, err)
assert.Nil(t, c.replaceOSD)
})
@@ -761,8 +760,9 @@ func TestReplaceOSDForNewStore(t *testing.T) {
d := getDummyDeploymentOnNode(clientset, c, "node2", 0)
d.Labels[osdStore] = "bluestore-rdr"
createDeploymentOrPanic(clientset, d)
err := c.replaceOSDForNewStore()
osdsToBeReplaced, err := c.replaceOSDForNewStore()
assert.NoError(t, err)
assert.Equal(t, 0, len(osdsToBeReplaced))
assert.Nil(t, c.replaceOSD)
})
@@ -779,7 +779,7 @@ func TestReplaceOSDForNewStore(t *testing.T) {
// create osd deployment with `bluestore` backend store
d := getDummyDeploymentOnNode(clientset, c, "node2", 1)
createDeploymentOrPanic(clientset, d)
err := c.replaceOSDForNewStore()
_, err := c.replaceOSDForNewStore()
assert.NoError(t, err)
assert.NotNil(t, c.replaceOSD)
assert.Equal(t, 1, c.replaceOSD.ID)
@@ -816,9 +816,9 @@ func TestReplaceOSDForNewStore(t *testing.T) {
c := New(context, clusterInfo, spec, "myversion")
d := getDummyDeploymentOnPVC(clientset, c, "pvc1", 2)
createDeploymentOrPanic(clientset, d)
err := c.replaceOSDForNewStore()
osdsToBeReplaced, err := c.replaceOSDForNewStore()
assert.NoError(t, err)
fmt.Printf("%+v", c.replaceOSD)
assert.Equal(t, 1, len(osdsToBeReplaced))
assert.NotNil(t, c.replaceOSD)
assert.Equal(t, 2, c.replaceOSD.ID)
assert.Equal(t, "pvc1", c.replaceOSD.Path)
@@ -862,8 +862,9 @@ func TestReplaceOSDForNewStore(t *testing.T) {
}
context.Executor = executor
c := New(context, clusterInfo, spec, "myversion")
err := c.replaceOSDForNewStore()
osdsToBeReplaced, err := c.replaceOSDForNewStore()
assert.NoError(t, err)
assert.Equal(t, 0, len(osdsToBeReplaced))
assert.Nil(t, c.replaceOSD)
})
}
+47 -29
View File
@@ -19,7 +19,6 @@ package osd
import (
"encoding/json"
"fmt"
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
@@ -43,6 +42,16 @@ type OSDReplaceInfo struct {
Node string `json:"node"`
}
type OSDReplaceInfoList []OSDReplaceInfo
func (o *OSDReplaceInfoList) getOSDIds() []int {
osdIDs := []int{}
for _, osd := range *o {
osdIDs = append(osdIDs, osd.ID)
}
return osdIDs
}
func (o *OSDReplaceInfo) saveAsConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error {
configStr, err := o.string()
if err != nil {
@@ -81,58 +90,67 @@ func (o *OSDReplaceInfo) string() (string, error) {
return string(configInBytes), nil
}
// getOSDReplaceInfo returns an existing OSD that needs to be replaced for a new backend store
func (c *Cluster) getOSDReplaceInfo() (*OSDReplaceInfo, error) {
osdReplaceInfo, err := GetOSDReplaceConfigMap(c.context, c.clusterInfo)
func (c *Cluster) replaceOSDForNewStore() (OSDReplaceInfoList, error) {
if c.spec.Storage.Store.UpdateStore != OSDStoreUpdateConfirmation {
logger.Info("no replacement of osds is requested")
return nil, nil
}
osdsToBeReplaced, err := c.getOSDWithNonMatchingStore()
if err != nil {
return nil, errors.Wrap(err, "failed to get any existing OSD in replace configmap")
return nil, errors.Wrapf(err, "failed to get information about the OSDs where backing store does not match the spec in namespace %q", c.clusterInfo.Namespace)
}
if osdReplaceInfo != nil {
return osdReplaceInfo, nil
if len(osdsToBeReplaced) == 0 {
logger.Debug("all OSDs are using the desired backing store. No replacement is required.")
return osdsToBeReplaced, nil
}
// replace an OSD only if Pgs are healthy
pgHealthMsg, pgClean, err := cephclient.IsClusterClean(c.context, c.clusterInfo, c.spec.DisruptionManagement.PGHealthyRegex)
if err != nil {
return nil, errors.Wrapf(err, "failed to check if the pgs are clean before replacing OSDs")
}
if !pgClean {
logger.Warningf("skipping OSD replacement because pgs are not healthy. PG status: %q", pgHealthMsg)
return nil, nil
return osdsToBeReplaced, nil
}
logger.Infof("placement group status: %q", pgHealthMsg)
osdsToReplace, err := c.getOSDWithNonMatchingStore()
// Check for an existing OSDs in the configmap
osdToBeReplaced, err := GetOSDReplaceConfigMap(c.context, c.clusterInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to list out OSDs with non matching backend store")
return nil, errors.Wrap(err, "failed to get any existing OSD in replace configmap")
}
if osdToBeReplaced != nil {
c.replaceOSD = osdToBeReplaced
} else {
c.replaceOSD = &osdsToBeReplaced[0]
}
if len(osdsToReplace) == 0 {
logger.Infof("all osds have already been migrated to backend store %q", c.spec.Storage.Store.Type)
return nil, nil
logger.Infof("replacing OSD.%d to the new backing store %q", c.replaceOSD.ID, c.spec.Storage.Store.Type)
err = c.deleteOSDDeployment(c.replaceOSD.ID)
if err != nil {
return nil, errors.Wrapf(err, "failed to delete OSD deployment that needs migration to new backend store in namespace %q", c.clusterInfo.Namespace)
}
logger.Infof("%d osd(s) require migration to new backend store %q.", len(osdsToReplace), c.spec.Storage.Store.Type)
return &osdsToReplace[0], nil
return osdsToBeReplaced, nil
}
// getOSDWithNonMatchingStore returns OSDs with osd-store label different from expected store in cephCluster spec
func (c *Cluster) getOSDWithNonMatchingStore() ([]OSDReplaceInfo, error) {
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)}
deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts)
if err != nil {
return nil, errors.Wrap(err, "failed to query OSDs to skip reconcile")
}
func (c *Cluster) getOSDWithNonMatchingStore() (OSDReplaceInfoList, error) {
osdReplaceList := []OSDReplaceInfo{}
for i := range deployments.Items {
if osdStore, ok := deployments.Items[i].Labels[osdStore]; ok {
// get existing OSD deployments
osdDeployments, err := c.getOSDDeployments()
if err != nil {
return osdReplaceList, errors.Wrapf(err, "failed to get existing OSD deployments in namespace %q", c.clusterInfo.Namespace)
}
for i := range osdDeployments.Items {
if osdStore, ok := osdDeployments.Items[i].Labels[osdStore]; ok {
if osdStore != string(c.spec.Storage.Store.Type) {
osdInfo, err := c.getOSDInfo(&deployments.Items[i])
osdInfo, err := c.getOSDInfo(&osdDeployments.Items[i])
if err != nil {
return nil, errors.Wrapf(err, "failed to details about the OSD %q", deployments.Items[i].Name)
return nil, errors.Wrapf(err, "failed to details about the OSD %q", osdDeployments.Items[i].Name)
}
var path string
if osdInfo.PVCName != "" {
@@ -21,12 +21,9 @@ import (
"context"
"testing"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/k8sutil"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/client-go/kubernetes/fake"
@@ -163,125 +160,3 @@ func TestGetOSDWithNonMatchingStoreOnPVCs(t *testing.T) {
assert.Equal(t, "pvc0", osdList[0].Path)
})
}
func TestGetOSDReplaceInfo(t *testing.T) {
namespace := "rook-ceph"
clientset := fake.NewSimpleClientset()
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
if args[0] == "status" {
return healthyCephStatus, nil
}
return "", errors.Errorf("unexpected ceph command '%v'", args)
},
}
ctx := &clusterd.Context{
Clientset: clientset,
Executor: executor,
}
clusterInfo := &cephclient.ClusterInfo{
Namespace: namespace,
Context: context.TODO(),
}
clusterInfo.SetName("mycluster")
clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
spec := cephv1.ClusterSpec{}
c := New(ctx, clusterInfo, spec, "rook/rook:master")
t.Run("no OSD replace info available", func(t *testing.T) {
actualOSDInfo, err := c.getOSDReplaceInfo()
assert.NoError(t, err)
assert.Nil(t, actualOSDInfo)
})
t.Run("ensure no OSD to replace if pgs are not healthy", func(t *testing.T) {
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
if args[0] == "status" {
return unHealthyCephStatus, nil
}
return "", errors.Errorf("unexpected ceph command '%v'", args)
},
}
ctx.Executor = executor
c := New(ctx, clusterInfo, spec, "rook/rook:master")
actualOSDInfo, err := c.getOSDReplaceInfo()
assert.NoError(t, err)
assert.Nil(t, actualOSDInfo)
})
t.Run("read OSD 0 replace info from config map", func(t *testing.T) {
actualOSDInfo := &OSDReplaceInfo{ID: 0, Path: "/dev/sda", Node: "node1"}
err := actualOSDInfo.saveAsConfig(c.context, c.clusterInfo)
assert.NoError(t, err)
expectedOSDInfo, err := c.getOSDReplaceInfo()
assert.NoError(t, err)
assert.NotNil(t, expectedOSDInfo)
assert.Equal(t, 0, expectedOSDInfo.ID)
err = k8sutil.DeleteConfigMap(clusterInfo.Context, ctx.Clientset, OSDReplaceConfigName, namespace, &k8sutil.DeleteOptions{})
assert.NoError(t, err)
})
t.Run("ensure no OSD replace info if all OSDs using expected OSD store", func(t *testing.T) {
spec := cephv1.ClusterSpec{
Storage: cephv1.StorageScopeSpec{
Store: cephv1.OSDStore{
Type: "bluestore-rdr",
UpdateStore: "yes-really-update-store",
},
},
}
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
if args[0] == "status" {
return healthyCephStatus, nil
}
return "", errors.Errorf("unexpected ceph command '%v'", args)
},
}
ctx.Executor = executor
c := New(ctx, clusterInfo, spec, "myversion")
d := getDummyDeploymentOnNode(clientset, c, "node2", 1)
d.Labels[osdStore] = "bluestore-rdr"
createDeploymentOrPanic(clientset, d)
expectedOSDInfo, err := c.getOSDReplaceInfo()
assert.NoError(t, err)
assert.Nil(t, expectedOSDInfo)
})
t.Run("ensure OSD replace info if OSD store is not up to date", func(t *testing.T) {
spec := cephv1.ClusterSpec{
Storage: cephv1.StorageScopeSpec{
Store: cephv1.OSDStore{
Type: "bluestore-rdr",
UpdateStore: "yes-really-update-store",
},
},
}
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
if args[0] == "status" {
return healthyCephStatus, nil
}
return "", errors.Errorf("unexpected ceph command '%v'", args)
},
}
ctx.Executor = executor
c := New(ctx, clusterInfo, spec, "myversion")
d := getDummyDeploymentOnNode(clientset, c, "node3", 2)
createDeploymentOrPanic(clientset, d)
expectedOSDInfo, err := c.getOSDReplaceInfo()
assert.NoError(t, err)
assert.NotNil(t, expectedOSDInfo)
assert.Equal(t, 2, expectedOSDInfo.ID)
assert.Equal(t, "node3", expectedOSDInfo.Node)
})
}
+20 -2
View File
@@ -232,7 +232,6 @@ func (c *Cluster) getOSDUpdateInfo(errs *provisionErrors) (*updateQueue, *existe
updateQueue := newUpdateQueueWithCapacity(len(deps.Items))
existenceList := newExistenceListWithCapacity(len(deps.Items))
for i := range deps.Items {
id, err := getOSDID(&deps.Items[i]) // avoid implicit memory aliasing by indexing
if err != nil {
@@ -240,7 +239,6 @@ func (c *Cluster) getOSDUpdateInfo(errs *provisionErrors) (*updateQueue, *existe
errs.addError("%v. did a user create their own deployment with label %q?", selector, err)
continue
}
// all OSD deployments should be marked as existing
existenceList.Add(id)
updateQueue.Push(id)
@@ -299,6 +297,11 @@ func (q *updateQueue) Exists(osdID int) bool {
return false
}
// Clear deletes all entries inside the queue
func (q *updateQueue) Clear() {
q.q = q.q[:0]
}
// Remove removes the items from the queue if they exist.
func (q *updateQueue) Remove(osdIDs []int) {
shouldRemove := func(rid int) bool {
@@ -369,3 +372,18 @@ func (c *Cluster) getFuncToListDeploymentsWithIDs(osdIDs []string) func() (*apps
return c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts)
}
}
// getOSDDeployments returns the list of existing OSD deployments
func (c *Cluster) getOSDDeployments() (*appsv1.DeploymentList, error) {
namespace := c.clusterInfo.Namespace
selector := fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)
listOpts := metav1.ListOptions{
// list only rook-ceph-osd Deployments
LabelSelector: selector,
}
deps, err := c.context.Clientset.AppsV1().Deployments(namespace).List(c.clusterInfo.Context, listOpts)
if err != nil {
return nil, errors.Wrap(err, "failed to query existing OSD deployments to check if they need to be updated")
}
return deps, nil
}