From d8428eeba94de17ee992c12dbc7cea7dd5f33dfe Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 23 Jul 2024 15:42:31 -0600 Subject: [PATCH] tests: retry helm upgrade during upgrade tests The helm upgrade tests have been failing frequently, but not always, on the oldest version of K8s that is tested in the CI for the past few months. Add a retry to attempt to get the CI passing more consistently. Signed-off-by: Travis Nielsen (cherry picked from commit 043b4466755c6f12a22e484cc2442205da979e31) --- tests/framework/utils/helm_helper.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/framework/utils/helm_helper.go b/tests/framework/utils/helm_helper.go index c94eb4c46..0a8323e7b 100644 --- a/tests/framework/utils/helm_helper.go +++ b/tests/framework/utils/helm_helper.go @@ -21,6 +21,7 @@ import ( "os" "path" "path/filepath" + "time" "gopkg.in/yaml.v2" @@ -99,11 +100,21 @@ func (h *HelmHelper) InstallLocalHelmChart(upgrade bool, namespace, chart string cmdArgs = append(cmdArgs, "--namespace", namespace) } - err = h.installChart(cmdArgs, values) - if err != nil { - return fmt.Errorf("failed to install local helm chart %s with in namespace: %v, err=%v", chart, namespace, err) + var lastErr error + maxRetries := 5 + for i := 0; i < maxRetries; i++ { + err = h.installChart(cmdArgs, values) + if err != nil { + lastErr = fmt.Errorf("failed to install local helm chart %s with in namespace: %v, err=%v", chart, namespace, err) + logger.Error(lastErr) + time.Sleep(time.Second) + continue + } + logger.Infof("helm chart %q installed successfully after %d attempt(s)", chart, i+1) + return nil } - return nil + logger.Errorf("failed to install helm chart %s after %d attempts", chart, maxRetries) + return lastErr } func (h *HelmHelper) InstallVersionedChart(namespace, chart, version string, values map[string]interface{}) error {