diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 1d464b594..feb15c66f 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -300,6 +300,28 @@ func (h *CephInstaller) waitForCluster() error { } } + // The CSI operator creates the driver deployments asynchronously, so they + // may still be pulling images when the ceph daemons are already up. If a + // PVC is created before the provisioner is ready, the first provisioning + // attempt fails and the external-provisioner sidecar backs off + // exponentially, delaying the volume creation by several minutes. + drivers := []string{"rbd", "cephfs"} + if h.settings.TestNFSCSI { + drivers = append(drivers, "nfs") + } + for _, driver := range drivers { + // Look up the deployment by name: the csi-operator does not set labels + // on the deployment object itself, only on its pod template. Wait for a + // single ready replica: the ctrlplugin runs two replicas with pod + // anti-affinity, so only one can be scheduled on a single-node CI + // cluster, and the provisioner sidecars use leader election so one + // ready replica is enough to provision volumes. + name := fmt.Sprintf("%s.%s.csi.ceph.com-ctrlplugin", h.settings.OperatorNamespace, driver) + if err := h.k8shelper.WaitForDeploymentReadyReplicas(name, h.settings.OperatorNamespace, 1, 2*utils.RetryLoop); err != nil { + return err + } + } + logger.Infof("Rook Cluster started") if !h.settings.SkipOSDCreation { return h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", h.settings.Namespace) diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index ffdb9f5c9..86af9f133 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -1571,6 +1571,23 @@ func (k8sh *K8sHelper) WaitForLabeledDeploymentsToBeReady(label, namespace strin return k8sh.WaitForLabeledDeploymentsToBeReadyWithRetries(label, namespace, RetryLoop) } +// WaitForDeploymentReadyReplicas waits for the named deployment to have at least minReady ready +// replicas. Retry the number of times given. +func (k8sh *K8sHelper) WaitForDeploymentReadyReplicas(name, namespace string, minReady int32, retries int) error { + ctx := context.TODO() + for i := 0; i < retries; i++ { + d, err := k8sh.Clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err == nil && d.Status.ReadyReplicas >= minReady { + logger.Infof("deployment %q in namespace %q has %d ready replicas", name, namespace, d.Status.ReadyReplicas) + return nil + } + logger.Infof("waiting for deployment %q in namespace %q to have %d ready replicas. ready=%d/%d, err=%v", + name, namespace, minReady, d.Status.ReadyReplicas, d.Status.Replicas, err) + time.Sleep(RetryInterval * time.Second) + } + return fmt.Errorf("giving up waiting for deployment %q in namespace %q to have %d ready replicas", name, namespace, minReady) +} + // WaitForLabeledDeploymentsToBeReadyWithRetries waits for all deployments matching the given label // selector to be fully ready. Retry the number of times given. func (k8sh *K8sHelper) WaitForLabeledDeploymentsToBeReadyWithRetries(label, namespace string, retries int) error {