diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 688d4c264..5c5fc7c91 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -186,6 +186,25 @@ func (k8sh *K8sHelper) KubectlWithStdin(stdin string, args ...string) (string, e } func getManifestFromURL(url string) (string, error) { + var lastErr error + // retry the download since fetches from raw.githubusercontent.com fail + // transiently in CI + for i := 0; i < 5; i++ { + if i > 0 { + time.Sleep(5 * time.Second) + } + manifest, err := downloadManifest(url) + if err != nil { + logger.Warningf("failed to get manifest, will retry. %v", err) + lastErr = err + continue + } + return manifest, nil + } + return "", lastErr +} + +func downloadManifest(url string) (string, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return "", err @@ -195,6 +214,9 @@ func getManifestFromURL(url string) (string, error) { return "", errors.Wrapf(err, "failed to get manifest from url %s", url) } defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", errors.Errorf("failed to get manifest from url %s. status: %s", url, res.Status) + } body, err := io.ReadAll(res.Body) if err != nil { return "", errors.Wrapf(err, "failed to read manifest from url %s", url) diff --git a/tests/framework/utils/snapshot.go b/tests/framework/utils/snapshot.go index 966a89810..60566a0fe 100644 --- a/tests/framework/utils/snapshot.go +++ b/tests/framework/utils/snapshot.go @@ -62,6 +62,21 @@ func (k8sh *K8sHelper) CheckSnapshotISReadyToUse(name, namespace string, retries return false, fmt.Errorf("giving up waiting for %q snapshot in namespace %q", name, namespace) } +// kubectlWithURLRetry runs kubectl with a manifest URL argument, retrying on +// failure since fetching manifests from raw.githubusercontent.com fails +// transiently in CI. +func (k8sh *K8sHelper) kubectlWithURLRetry(args ...string) error { + var err error + for i := 0; i < 5; i++ { + if _, err = k8sh.Kubectl(args...); err == nil { + return nil + } + logger.Warningf("failed to run kubectl %v, will retry. %v", args, err) + time.Sleep(5 * time.Second) + } + return err +} + // snapshotController creates/applies or deletes the snapshotcontroller and required RBAC func (k8sh *K8sHelper) snapshotController(action string) error { controllerURL := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, controllerPath) @@ -78,11 +93,7 @@ func (k8sh *K8sHelper) snapshotController(action string) error { } rbac := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, rbacPath) - _, err = k8sh.Kubectl(action, "-f", rbac) - if err != nil { - return err - } - return nil + return k8sh.kubectlWithURLRetry(action, "-f", rbac) } // WaitForSnapshotController check snapshotcontroller is ready within given @@ -130,39 +141,19 @@ func (k8sh *K8sHelper) snapshotCRD(action string) error { } return a } - snapshotClassCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, snapshotClassCRDPath) - _, err := k8sh.Kubectl(args(snapshotClassCRD)...) - if err != nil { - return err + crdPaths := []string{ + snapshotClassCRDPath, + volumeSnapshotContentsCRDPath, + volumeSnapshotCRDPath, + volumeGroupSnapshotClassCRDPath, + volumeGroupSnapshotContentsCRDPath, + volumeGroupSnapshotCRDPath, } - - snapshotContentsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeSnapshotContentsCRDPath) - _, err = k8sh.Kubectl(args(snapshotContentsCRD)...) - if err != nil { - return err - } - - snapshotCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeSnapshotCRDPath) - _, err = k8sh.Kubectl(args(snapshotCRD)...) - if err != nil { - return err - } - vgsClassCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotClassCRDPath) - _, err = k8sh.Kubectl(args(vgsClassCRD)...) - if err != nil { - return err - } - - vgsContentsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotContentsCRDPath) - _, err = k8sh.Kubectl(args(vgsContentsCRD)...) - if err != nil { - return err - } - - vgsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotCRDPath) - _, err = k8sh.Kubectl(args(vgsCRD)...) - if err != nil { - return err + for _, crdPath := range crdPaths { + crdURL := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, crdPath) + if err := k8sh.kubectlWithURLRetry(args(crdURL)...); err != nil { + return err + } } return nil diff --git a/tests/integration/ceph_base_file_test.go b/tests/integration/ceph_base_file_test.go index 28a7290d8..1d756f9a4 100644 --- a/tests/integration/ceph_base_file_test.go +++ b/tests/integration/ceph_base_file_test.go @@ -151,7 +151,8 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper require.NoError(s.T(), err) }() logger.Infof("check snapshot controller is running") - err = k8sh.WaitForSnapshotController(15) + // the snapshot-controller image pull can take a few minutes in CI + err = k8sh.WaitForSnapshotController(90) require.NoError(s.T(), err) // create snapshot class snapshotDeletePolicy := "Delete"