From 36de8f7c97b49ef1e3f91c2b9969fca48535f418 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Thu, 11 Jun 2026 17:13:01 -0700 Subject: [PATCH] ci: wait for csi driver deployments after cluster install The csi-operator creates the driver deployments asynchronously, so the ctrlplugin pods may still be pulling images when the ceph daemons are already up and the integration tests start. If a PVC is created before the provisioner is ready, the first provisioning attempt fails (the mounted csi config may also not be visible to the driver yet) and the external-provisioner sidecar enters exponential backoff, delaying the volume creation by several minutes and timing out the test wait. Observed in TestCephSmokeSuite runs as 'timed out waiting for image count to reach 1' with the rbd ctrlplugin pod cold-starting mid-test and the first CreateVolume failing with 'failed to fetch monitor list using clusterID'. In one run the second provisioning attempt landed two seconds after the test wait expired. Wait for the ctrlplugin deployments of the enabled drivers to be ready before declaring the cluster installed. Signed-off-by: Joshua Hoblitt (cherry picked from commit 53ab1c513648d3d8b053c7ddc8a05bb828dad6c1) --- tests/framework/installer/ceph_installer.go | 22 +++++++++++++++++++++ tests/framework/utils/k8s_helper.go | 17 ++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 1d464b594..feb15c66f 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -300,6 +300,28 @@ func (h *CephInstaller) waitForCluster() error { } } + // The CSI operator creates the driver deployments asynchronously, so they + // may still be pulling images when the ceph daemons are already up. If a + // PVC is created before the provisioner is ready, the first provisioning + // attempt fails and the external-provisioner sidecar backs off + // exponentially, delaying the volume creation by several minutes. + drivers := []string{"rbd", "cephfs"} + if h.settings.TestNFSCSI { + drivers = append(drivers, "nfs") + } + for _, driver := range drivers { + // Look up the deployment by name: the csi-operator does not set labels + // on the deployment object itself, only on its pod template. Wait for a + // single ready replica: the ctrlplugin runs two replicas with pod + // anti-affinity, so only one can be scheduled on a single-node CI + // cluster, and the provisioner sidecars use leader election so one + // ready replica is enough to provision volumes. + name := fmt.Sprintf("%s.%s.csi.ceph.com-ctrlplugin", h.settings.OperatorNamespace, driver) + if err := h.k8shelper.WaitForDeploymentReadyReplicas(name, h.settings.OperatorNamespace, 1, 2*utils.RetryLoop); err != nil { + return err + } + } + logger.Infof("Rook Cluster started") if !h.settings.SkipOSDCreation { return h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", h.settings.Namespace) diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index ffdb9f5c9..86af9f133 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -1571,6 +1571,23 @@ func (k8sh *K8sHelper) WaitForLabeledDeploymentsToBeReady(label, namespace strin return k8sh.WaitForLabeledDeploymentsToBeReadyWithRetries(label, namespace, RetryLoop) } +// WaitForDeploymentReadyReplicas waits for the named deployment to have at least minReady ready +// replicas. Retry the number of times given. +func (k8sh *K8sHelper) WaitForDeploymentReadyReplicas(name, namespace string, minReady int32, retries int) error { + ctx := context.TODO() + for i := 0; i < retries; i++ { + d, err := k8sh.Clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err == nil && d.Status.ReadyReplicas >= minReady { + logger.Infof("deployment %q in namespace %q has %d ready replicas", name, namespace, d.Status.ReadyReplicas) + return nil + } + logger.Infof("waiting for deployment %q in namespace %q to have %d ready replicas. ready=%d/%d, err=%v", + name, namespace, minReady, d.Status.ReadyReplicas, d.Status.Replicas, err) + time.Sleep(RetryInterval * time.Second) + } + return fmt.Errorf("giving up waiting for deployment %q in namespace %q to have %d ready replicas", name, namespace, minReady) +} + // WaitForLabeledDeploymentsToBeReadyWithRetries waits for all deployments matching the given label // selector to be fully ready. Retry the number of times given. func (k8sh *K8sHelper) WaitForLabeledDeploymentsToBeReadyWithRetries(label, namespace string, retries int) error {