ci: retry snapshot controller and crd manifest fetches

Installing the external-snapshotter CRDs and controller runs several
kubectl commands against raw.githubusercontent.com manifest URLs with
no retry, and getManifestFromURL did not retry either, nor did it check
the HTTP status code, so an error page could be fed to kubectl as a
manifest. Transient fetch failures showed up in TestCephSmokeSuite as
NFS test failures during snapshot CRD installation.

Retry the URL-based kubectl invocations and the manifest download, fail
on non-200 responses, and bump the snapshot controller wait in the file
test from 75s to 450s to match the block test since the
snapshot-controller image pull can take longer in CI.

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>

tests/framework/utils/k8s_helper.go

@@ -186,6 +186,25 @@ func (k8sh *K8sHelper) KubectlWithStdin(stdin string, args ...string) (string, e
 }
 
 func getManifestFromURL(url string) (string, error) {
+	var lastErr error
+	// retry the download since fetches from raw.githubusercontent.com fail
+	// transiently in CI
+	for i := 0; i < 5; i++ {
+		if i > 0 {
+			time.Sleep(5 * time.Second)
+		}
+		manifest, err := downloadManifest(url)
+		if err != nil {
+			logger.Warningf("failed to get manifest, will retry. %v", err)
+			lastErr = err
+			continue
+		}
+		return manifest, nil
+	}
+	return "", lastErr
+}
+
+func downloadManifest(url string) (string, error) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return "", err
@@ -195,6 +214,9 @@ func getManifestFromURL(url string) (string, error) {
 		return "", errors.Wrapf(err, "failed to get manifest from url %s", url)
 	}
 	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", errors.Errorf("failed to get manifest from url %s. status: %s", url, res.Status)
+	}
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to read manifest from url %s", url)

tests/framework/utils/snapshot.go

@@ -62,6 +62,21 @@ func (k8sh *K8sHelper) CheckSnapshotISReadyToUse(name, namespace string, retries
 	return false, fmt.Errorf("giving up waiting for %q snapshot in namespace %q", name, namespace)
 }
 
+// kubectlWithURLRetry runs kubectl with a manifest URL argument, retrying on
+// failure since fetching manifests from raw.githubusercontent.com fails
+// transiently in CI.
+func (k8sh *K8sHelper) kubectlWithURLRetry(args ...string) error {
+	var err error
+	for i := 0; i < 5; i++ {
+		if _, err = k8sh.Kubectl(args...); err == nil {
+			return nil
+		}
+		logger.Warningf("failed to run kubectl %v, will retry. %v", args, err)
+		time.Sleep(5 * time.Second)
+	}
+	return err
+}
+
 // snapshotController creates/applies or deletes the snapshotcontroller and required RBAC
 func (k8sh *K8sHelper) snapshotController(action string) error {
 	controllerURL := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, controllerPath)
@@ -78,11 +93,7 @@ func (k8sh *K8sHelper) snapshotController(action string) error {
 	}
 
 	rbac := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, rbacPath)
-	_, err = k8sh.Kubectl(action, "-f", rbac)
-	if err != nil {
-		return err
-	}
-	return nil
+	return k8sh.kubectlWithURLRetry(action, "-f", rbac)
 }
 
 // WaitForSnapshotController check snapshotcontroller is ready within given
@@ -130,39 +141,19 @@ func (k8sh *K8sHelper) snapshotCRD(action string) error {
 		}
 		return a
 	}
-	snapshotClassCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, snapshotClassCRDPath)
-	_, err := k8sh.Kubectl(args(snapshotClassCRD)...)
-	if err != nil {
-		return err
+	crdPaths := []string{
+		snapshotClassCRDPath,
+		volumeSnapshotContentsCRDPath,
+		volumeSnapshotCRDPath,
+		volumeGroupSnapshotClassCRDPath,
+		volumeGroupSnapshotContentsCRDPath,
+		volumeGroupSnapshotCRDPath,
 	}
-
-	snapshotContentsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeSnapshotContentsCRDPath)
-	_, err = k8sh.Kubectl(args(snapshotContentsCRD)...)
-	if err != nil {
-		return err
-	}
-
-	snapshotCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeSnapshotCRDPath)
-	_, err = k8sh.Kubectl(args(snapshotCRD)...)
-	if err != nil {
-		return err
-	}
-	vgsClassCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotClassCRDPath)
-	_, err = k8sh.Kubectl(args(vgsClassCRD)...)
-	if err != nil {
-		return err
-	}
-
-	vgsContentsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotContentsCRDPath)
-	_, err = k8sh.Kubectl(args(vgsContentsCRD)...)
-	if err != nil {
-		return err
-	}
-
-	vgsCRD := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, volumeGroupSnapshotCRDPath)
-	_, err = k8sh.Kubectl(args(vgsCRD)...)
-	if err != nil {
-		return err
+	for _, crdPath := range crdPaths {
+		crdURL := fmt.Sprintf("%s/%s/%s", repoURL, snapshotterVersion, crdPath)
+		if err := k8sh.kubectlWithURLRetry(args(crdURL)...); err != nil {
+			return err
+		}
 	}
 
 	return nil

tests/integration/ceph_base_file_test.go

@@ -151,7 +151,8 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper
 		require.NoError(s.T(), err)
 	}()
 	logger.Infof("check snapshot controller is running")
-	err = k8sh.WaitForSnapshotController(15)
+	// the snapshot-controller image pull can take a few minutes in CI
+	err = k8sh.WaitForSnapshotController(90)
 	require.NoError(s.T(), err)
 	// create snapshot class
 	snapshotDeletePolicy := "Delete"