test: improve reliability of canary rgw-multisite-testing

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
(cherry picked from commit 92d9f994c2)

.github/workflows/canary-integration-test.yml

@@ -1549,8 +1549,8 @@ jobs:
         run: |
           tests/scripts/github-action-helper.sh deploy_first_rook_cluster
           kubectl create -f deploy/examples/object-multisite-test.yaml
-          # wait for multisite store to be created
-          tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph
+          # wait for multisite-store to be created
+          tests/scripts/github-action-helper.sh wait_for cephobjectstore multisite-store rook-ceph 480
 
       - name: prep second cluster pull realm config
         shell: bash --noprofile --norc -eo pipefail -x {0}
@@ -1569,13 +1569,13 @@ jobs:
           tests/scripts/github-action-helper.sh deploy_second_rook_cluster
           kubectl create -f deploy/examples/object-multisite-pull-realm-test.yaml
           # wait for realms to be pulled and zone-b-multisite-store to be created
-          tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph-secondary
+          tests/scripts/github-action-helper.sh wait_for cephobjectstore zone-b-multisite-store rook-ceph-secondary 480
 
-      - name: wait for ceph cluster 1 to be ready
+      - name: wait for both ceph clusters to be ready
         shell: bash --noprofile --norc -eo pipefail -x {0}
         run: |
-          tests/scripts/validate_cluster.sh osd 1
-          kubectl -n rook-ceph get pods
+          tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph
+          tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph-secondary
 
       - name: write an object to one cluster, read from the other
         shell: bash --noprofile --norc -eo pipefail -x {0}

tests/scripts/github-action-helper.sh

@@ -470,23 +470,28 @@ function deploy_second_rook_cluster() {
   deploy_toolbox
 }
 
-function wait_for_rgw() {
-  for _ in {1..120}; do
-    if [ "$(kubectl -n "$1" get pod -l app=rook-ceph-rgw --no-headers --field-selector=status.phase=Running | wc -l)" -ge 1 ]; then
-      echo "rgw pod is found"
-      break
+function wait_for() {
+  local kind=${1?kind is required}
+  local name=${2?resource name is required}
+  local ns=${3:-rook-ceph}
+  local timeout=${4:-120}
+  local status=${5:-Ready}
+
+  local start_time="${SECONDS}"
+  local elapsed_time=0
+  while [[ $elapsed_time -lt $timeout ]]; do
+    if [[ "$(kubectl -n "$ns" get "$kind" "$name" -o 'jsonpath={..status.phase}')" == "$status" ]]; then
+      echo "${kind}/${name} in ${ns} is ${status} -  elapsed time ${elapsed_time}s"
+      return 0
     fi
-    echo "waiting for rgw pods"
-    sleep 5
-  done
-  for _ in {1..120}; do
-    if [ "$(kubectl -n "$1" get deployment -l app=rook-ceph-rgw -o yaml | yq read - 'items[0].status.readyReplicas')" -ge 1 ]; then
-      echo "rgw is ready"
-      break
-    fi
-    echo "waiting for rgw becomes ready"
+
+    elapsed_time=$((SECONDS - start_time))
+    echo "waiting for ${kind}/${name} in ${ns} to be ${status} - elapsed time ${elapsed_time}s"
     sleep 5
   done
+
+  echo "timed out waiting for ${kind}/${name} in ${ns} to be ${status} - elapsed time ${elapsed_time}s " >&2
+  exit 1
 }
 
 function verify_operator_log_message() {
@@ -555,7 +560,7 @@ function write_object_read_from_replica_cluster() {
   # a direct sub-shell.
   S3CMD_ERROR=0
   (
-    sleep 60
+    sleep 300
     kill -s SIGUSR1 $$
   ) 2>/dev/null &
   trap "{ S3CMD_ERROR=1; break; }" SIGUSR1