ci: wait for csi driver deployments after cluster install
The csi-operator creates the driver deployments asynchronously, so the
ctrlplugin pods may still be pulling images when the ceph daemons are
already up and the integration tests start. If a PVC is created before
the provisioner is ready, the first provisioning attempt fails (the
mounted csi config may also not be visible to the driver yet) and the
external-provisioner sidecar enters exponential backoff, delaying the
volume creation by several minutes and timing out the test wait.
Observed in TestCephSmokeSuite runs as 'timed out waiting for image
count to reach 1' with the rbd ctrlplugin pod cold-starting mid-test
and the first CreateVolume failing with 'failed to fetch monitor list
using clusterID'. In one run the second provisioning attempt landed two
seconds after the test wait expired.
Wait for the ctrlplugin deployments of the enabled drivers to be ready
before declaring the cluster installed.
Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
(cherry picked from commit 53ab1c5136)
This commit is contained in:
1 parent
c6247f44a5
commit
36de8f7c97
2 files changed
+39
No files matched your search
@@ -300,6 +300,28 @@ func (h *CephInstaller) waitForCluster() error {
|
||||
}
|
||||
}
|
||||
|
||||
// The CSI operator creates the driver deployments asynchronously, so they
|
||||
// may still be pulling images when the ceph daemons are already up. If a
|
||||
// PVC is created before the provisioner is ready, the first provisioning
|
||||
// attempt fails and the external-provisioner sidecar backs off
|
||||
// exponentially, delaying the volume creation by several minutes.
|
||||
drivers := []string{"rbd", "cephfs"}
|
||||
if h.settings.TestNFSCSI {
|
||||
drivers = append(drivers, "nfs")
|
||||
}
|
||||
for _, driver := range drivers {
|
||||
// Look up the deployment by name: the csi-operator does not set labels
|
||||
// on the deployment object itself, only on its pod template. Wait for a
|
||||
// single ready replica: the ctrlplugin runs two replicas with pod
|
||||
// anti-affinity, so only one can be scheduled on a single-node CI
|
||||
// cluster, and the provisioner sidecars use leader election so one
|
||||
// ready replica is enough to provision volumes.
|
||||
name := fmt.Sprintf("%s.%s.csi.ceph.com-ctrlplugin", h.settings.OperatorNamespace, driver)
|
||||
if err := h.k8shelper.WaitForDeploymentReadyReplicas(name, h.settings.OperatorNamespace, 1, 2*utils.RetryLoop); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("Rook Cluster started")
|
||||
if !h.settings.SkipOSDCreation {
|
||||
return h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", h.settings.Namespace)
|
||||
|
||||
@@ -1571,6 +1571,23 @@ func (k8sh *K8sHelper) WaitForLabeledDeploymentsToBeReady(label, namespace strin
|
||||
return k8sh.WaitForLabeledDeploymentsToBeReadyWithRetries(label, namespace, RetryLoop)
|
||||
}
|
||||
|
||||
// WaitForDeploymentReadyReplicas waits for the named deployment to have at least minReady ready
|
||||
// replicas. Retry the number of times given.
|
||||
func (k8sh *K8sHelper) WaitForDeploymentReadyReplicas(name, namespace string, minReady int32, retries int) error {
|
||||
ctx := context.TODO()
|
||||
for i := 0; i < retries; i++ {
|
||||
d, err := k8sh.Clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
|
||||
if err == nil && d.Status.ReadyReplicas >= minReady {
|
||||
logger.Infof("deployment %q in namespace %q has %d ready replicas", name, namespace, d.Status.ReadyReplicas)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("waiting for deployment %q in namespace %q to have %d ready replicas. ready=%d/%d, err=%v",
|
||||
name, namespace, minReady, d.Status.ReadyReplicas, d.Status.Replicas, err)
|
||||
time.Sleep(RetryInterval * time.Second)
|
||||
}
|
||||
return fmt.Errorf("giving up waiting for deployment %q in namespace %q to have %d ready replicas", name, namespace, minReady)
|
||||
}
|
||||
|
||||
// WaitForLabeledDeploymentsToBeReadyWithRetries waits for all deployments matching the given label
|
||||
// selector to be fully ready. Retry the number of times given.
|
||||
func (k8sh *K8sHelper) WaitForLabeledDeploymentsToBeReadyWithRetries(label, namespace string, retries int) error {
|
||||
|
||||
Reference in new issue
Block a user