From 98bd03a42064ba6724baa41d1ddf68cc002cd4af Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 29 Apr 2025 13:53:43 -0600 Subject: [PATCH] ci: extend timeout for mons in upgrade test When upgrading from one Ceph version to another, the new image to upgrade to can take a long time to pull in some cases before the upgrade can even begin. For example, some ceph-ci images regularly take 6+ minutes to pull, which exceeds the timeout waiting for mons to be ready. Extend the timeout for mons to account for these cases. Signed-off-by: Blaine Gardner --- tests/framework/utils/k8s_helper.go | 4 ++-- tests/integration/ceph_upgrade_test.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index cda2e672e..1652482d0 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -1527,13 +1527,13 @@ func IsKubectlErrorNotFound(output string, err error) bool { // WaitForDeploymentCount waits until the desired number of deployments with the label exist. The // deployments are not guaranteed to be running, only existing. func (k8sh *K8sHelper) WaitForDeploymentCount(label, namespace string, count int) error { - return k8sh.waitForDeploymentCountWithRetries(label, namespace, count, RetryLoop) + return k8sh.WaitForDeploymentCountWithRetries(label, namespace, count, RetryLoop) } // WaitForDeploymentCountWithRetries waits until the desired number of deployments with the label // exist, retrying the specified number of times. The deployments are not guaranteed to be running, // only existing. -func (k8sh *K8sHelper) waitForDeploymentCountWithRetries(label, namespace string, count, retries int) error { +func (k8sh *K8sHelper) WaitForDeploymentCountWithRetries(label, namespace string, count, retries int) error { ctx := context.TODO() options := metav1.ListOptions{LabelSelector: label} for i := 0; i < retries; i++ { diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 0aae3798a..7fd081196 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -364,7 +364,8 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) { func (s *UpgradeSuite) waitForUpgradedDaemons(previousVersion, versionLabel string, numOSDs int, waitForMDS bool) { // wait for the mon(s) to be updated monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,%s!=%s", versionLabel, previousVersion) - err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.settings.Mons) + // extend the timeout for this check because it can take 6+ minutes to pull ceph-ci images + err := s.k8sh.WaitForDeploymentCountWithRetries(monsNotOldVersion, s.namespace, s.settings.Mons, utils.RetryLoop+120) require.NoError(s.T(), err, "mon(s) didn't update") err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace) require.NoError(s.T(), err)