From c85643692e4104d27f1a231724886c69a07c723b Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Thu, 11 Jun 2026 17:13:16 -0700 Subject: [PATCH] ci: retry snapshot controller and crd manifest fetches Installing the external-snapshotter CRDs and controller runs several kubectl commands against raw.githubusercontent.com manifest URLs with no retry, and getManifestFromURL did not retry either, nor did it check the HTTP status code, so an error page could be fed to kubectl as a manifest. Transient fetch failures showed up in TestCephSmokeSuite as NFS test failures during snapshot CRD installation. Retry the URL-based kubectl invocations and the manifest download, fail on non-200 responses, and bump the snapshot controller wait in the file test from 75s to 450s to match the block test since the snapshot-controller image pull can take longer in CI. Signed-off-by: Joshua Hoblitt --- tests/framework/utils/k8s_helper.go | 22 ++++++++ tests/framework/utils/snapshot.go | 65 ++++++++++-------------- tests/integration/ceph_base_file_test.go | 3 +- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 688d4c264..5c5fc7c91 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -186,6 +186,25 @@ func (k8sh *K8sHelper) KubectlWithStdin(stdin string, args ...string) (string, e } func getManifestFromURL(url string) (string, error) { + var lastErr error + // retry the download since fetches from raw.githubusercontent.com fail + // transiently in CI + for i := 0; i < 5; i++ { + if i > 0 { + time.Sleep(5 * time.Second) + } + manifest, err := downloadManifest(url) + if err != nil { + logger.Warningf("failed to get manifest, will retry. %v", err) + lastErr = err + continue + } + return manifest, nil + } + return "", lastErr +} + +func downloadManifest(url string) (string, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return "", err @@ -195,6 +214,9 @@ func getManifestFromURL(url string) (string, error) { return "", errors.Wrapf(err, "failed to get manifest from url %s", url) } defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", errors.Errorf("failed to get manifest from url %s. status: %s", url, res.Status) + } body, err := io.ReadAll(res.Body) if err != nil { return "", errors.Wrapf(err, "failed to read manifest from url %s", url) diff --git a/tests/framework/utils/snapshot.go b/tests/framework/utils/snapshot.go index 966a89810..60566a0fe 100644 --- a/tests/framework/utils/snapshot.go +++ b/tests/framework/utils/snapshot.go @@ -62,6 +62,21 @@ func (k8sh *K8sHelper) CheckSnapshotISReadyToUse(name, namespace string, retries return false, fmt.Errorf("giving up waiting for %q snapshot in namespace %q", name, namespace) } +// kubectlWithURLRetry runs kubectl with a manifest URL argument, retrying on +// failure since fetching manifests from raw.githubusercontent.com fails +// transiently in CI. +func (k8sh *K8sHelper) kubectlWithURLRetry(args ...string) error { + var err error + for i := 0; i < 5; i++ { + if _, err = k8sh.Kubectl(args...); err == nil { + return nil + } + logger.Warningf("failed to run kubectl %v, will retry. %v", args, err) + time.Sleep(5 * time.Second) + } + return err +} + // snapshotController creates/applies or deletes the snapshotcontroller and required RBAC func (k8sh *K8sHelper) snapshotController(action string) error { controllerURL := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, controllerPath) @@ -78,11 +93,7 @@ func (k8sh *K8sHelper) snapshotController(action string) error { } rbac := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, rbacPath) - _, err = k8sh.Kubectl(action, "-f", rbac) - if err != nil { - return err - } - return nil + return k8sh.kubectlWithURLRetry(action, "-f", rbac) } // WaitForSnapshotController check snapshotcontroller is ready within given @@ -130,39 +141,19 @@ func (k8sh *K8sHelper) snapshotCRD(action string) error { } return a } - snapshotClassCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, snapshotClassCRDPath) - _, err := k8sh.Kubectl(args(snapshotClassCRD)...) - if err != nil { - return err + crdPaths := []string{ + snapshotClassCRDPath, + volumeSnapshotContentsCRDPath, + volumeSnapshotCRDPath, + volumeGroupSnapshotClassCRDPath, + volumeGroupSnapshotContentsCRDPath, + volumeGroupSnapshotCRDPath, } - - snapshotContentsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeSnapshotContentsCRDPath) - _, err = k8sh.Kubectl(args(snapshotContentsCRD)...) - if err != nil { - return err - } - - snapshotCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeSnapshotCRDPath) - _, err = k8sh.Kubectl(args(snapshotCRD)...) - if err != nil { - return err - } - vgsClassCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotClassCRDPath) - _, err = k8sh.Kubectl(args(vgsClassCRD)...) - if err != nil { - return err - } - - vgsContentsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotContentsCRDPath) - _, err = k8sh.Kubectl(args(vgsContentsCRD)...) - if err != nil { - return err - } - - vgsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotCRDPath) - _, err = k8sh.Kubectl(args(vgsCRD)...) - if err != nil { - return err + for _, crdPath := range crdPaths { + crdURL := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, crdPath) + if err := k8sh.kubectlWithURLRetry(args(crdURL)...); err != nil { + return err + } } return nil diff --git a/tests/integration/ceph_base_file_test.go b/tests/integration/ceph_base_file_test.go index 28a7290d8..1d756f9a4 100644 --- a/tests/integration/ceph_base_file_test.go +++ b/tests/integration/ceph_base_file_test.go @@ -151,7 +151,8 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper require.NoError(s.T(), err) }() logger.Infof("check snapshot controller is running") - err = k8sh.WaitForSnapshotController(15) + // the snapshot-controller image pull can take a few minutes in CI + err = k8sh.WaitForSnapshotController(90) require.NoError(s.T(), err) // create snapshot class snapshotDeletePolicy := "Delete"