From 43d3bf147aee36720098e1d653276b0ecbdcfcc1 Mon Sep 17 00:00:00 2001 From: sp98 Date: Fri, 19 Jan 2024 12:50:33 +0530 Subject: [PATCH] osd: wait for the pgs to be clean If migration of the OSDs is enabled, operator should wait for the pgs to be clean after the all the OSDs have reconciled. This will help add the exponential backoff for the operator restarts when OSD migration is needed but PGs are still rebalancing the data after last OSD migration. Signed-off-by: sp98 --- pkg/operator/ceph/cluster/osd/osd.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 0487ad25e..812cc5e4f 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -268,20 +268,30 @@ func (c *Cluster) Start() error { return errors.Wrapf(err, "failed to update ceph storage status") } - if c.replaceOSD != nil { - delOpts := &k8sutil.DeleteOptions{MustDelete: true, WaitOptions: k8sutil.WaitOptions{Wait: true}} - err := k8sutil.DeleteConfigMap(c.clusterInfo.Context, c.context.Clientset, OSDReplaceConfigName, namespace, delOpts) - if err != nil { - return errors.Wrapf(err, "failed to delete the %q configmap", OSDReplaceConfigName) + if c.spec.Storage.Store.UpdateStore == OSDStoreUpdateConfirmation { + if c.replaceOSD != nil { + delOpts := &k8sutil.DeleteOptions{MustDelete: true, WaitOptions: k8sutil.WaitOptions{Wait: true}} + err := k8sutil.DeleteConfigMap(c.clusterInfo.Context, c.context.Clientset, OSDReplaceConfigName, namespace, delOpts) + if err != nil { + return errors.Wrapf(err, "failed to delete the %q configmap", OSDReplaceConfigName) + } } - // Wait for PGs to be healthy before continuing the reconcile - _, err = c.waitForHealthyPGs() + // wait for the pgs to be healthy before attempting to migrate the next OSD + _, err := c.waitForHealthyPGs() if err != nil { return errors.Wrapf(err, "failed to wait for pgs to be healhty") } - return errors.New("reconcile operator to replace OSDs that are pending migration") + // reconcile if migration of one or more OSD is pending. + osdsToReplace, err := c.getOSDWithNonMatchingStore() + if err != nil { + return errors.Wrapf(err, "failed to check if any OSD migration is pending") + } + + if len(osdsToReplace) != 0 { + return errors.New("reconcile operator to replace OSDs that are pending migration") + } } logger.Infof("finished running OSDs in namespace %q", namespace) @@ -846,7 +856,7 @@ func (c *Cluster) waitForHealthyPGs() (bool, error) { if pgClean { return true, nil } - logger.Infof("waiting for PGs to be healthy after replacing an OSD, status: %q", pgHealthMsg) + logger.Infof("waiting for PGs to be healthy. PG status: %q", pgHealthMsg) return false, nil }