From bf008210568df443e615bb51a06b5d21c21ecb1b Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Wed, 10 Jun 2026 15:42:51 -0700 Subject: [PATCH] ci: make rgw-multisite-testing wait for multisite readiness The rgw-multisite-testing canary job has been chronically flaky (roughly a third of recent canary runs, including on master and release branches). Job logs and collected artifacts show two recurring mechanisms, neither of which is addressed by bumping timeouts: 1. Every recent failure of the "write an object" step died within ~16s: the very first "s3cmd mb" against the primary zone got ECONNREFUSED three times across the 10s retry window. The RGW process never crashed; RGWs pause their HTTP frontends to reload the realm whenever the RGW configuration period changes ("rgw realm reloader: Pausing frontends for realm update..."), which keeps happening shortly after the second zone joins the zonegroup, exactly when the test starts writing. While paused, the readiness probe also times out, the endpoint is removed from the Service, and connections to the ClusterIP are refused. 2. The "kubectl wait --for=condition=available deployment/..." barrier added previously is a no-op for these deployments: with one replica and maxUnavailable=1, the availability threshold is zero ready pods, so the condition is always true. Changes: - Wait on RGW pod readiness instead of deployment availability. - Add a "wait for multisite sync to be established" step that polls "radosgw-admin sync status" on both clusters until metadata and data sync report caught up, mirroring the checkpoints ceph's own multisite QA performs before asserting replication. - Probe both RGW endpoints over HTTP until several consecutive requests succeed before writing, to ride out realm-reload pauses. - Replace the fixed 3-attempt s3cmd retries with time-budgeted retries. - Replace the s3cmd read polling loop and its SIGUSR1 timer (which was never cancelled on success, silently shrinking the second direction's time budget) with "radosgw-admin bucket sync checkpoint", the marker-based barrier used by ceph QA, followed by a single get and diff. - Dump multisite diagnostics (pod state, RGW logs, sync status, period) when the job fails. Signed-off-by: Joshua Hoblitt --- .github/workflows/canary-integration-test.yml | 18 ++- tests/scripts/github-action-helper.sh | 141 +++++++++++++++--- 2 files changed, 133 insertions(+), 26 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 343404a00..e0a1a56fb 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1937,9 +1937,15 @@ jobs: run: | tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph-secondary - # wait for the RGW deployments to be ready to serve traffic - kubectl -n rook-ceph wait --for=condition=available deployment/rook-ceph-rgw-multisite-store-a --timeout=300s - kubectl -n rook-ceph-secondary wait --for=condition=available deployment/rook-ceph-rgw-zone-b-multisite-store-a --timeout=300s + # Wait for the RGW pods to be ready. With a single replica and maxUnavailable=1, the + # RGW deployments count as "available" even with zero ready pods, so wait on the pods. + kubectl -n rook-ceph wait --for=condition=Ready pod -l rook_object_store=multisite-store --timeout=300s + kubectl -n rook-ceph-secondary wait --for=condition=Ready pod -l rook_object_store=zone-b-multisite-store --timeout=300s + + - name: wait for multisite sync to be established + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh wait_for_multisite_sync_established - name: write an object to one cluster, read from the other shell: bash --noprofile --norc -eo pipefail -x {0} @@ -1962,6 +1968,12 @@ jobs: tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary_object_zone}" 600 tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary_object_zone}" 600 + - name: dump multisite diagnostics on failure + if: failure() + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh dump_multisite_diagnostics + - name: collect common logs if: always() uses: ./.github/workflows/collect-logs diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 5ee3fe905..8cf0d8b24 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -640,10 +640,108 @@ function retry() { exit 1 } +function retry_for() { + local timeout_seconds=${1?timeout in seconds is required} + shift + local deadline=$((SECONDS + timeout_seconds)) + while true; do + if "$@"; then return 0; fi + if [[ $SECONDS -ge $deadline ]]; then + echo "$* failed after retrying for ${timeout_seconds}s" + return 1 + fi + echo "$* failed, retrying in 5s..." + sleep 5 + done +} + +function wait_for_rgw_http() { + local endpoint_ip=${1?rgw endpoint ip is required} + local timeout_seconds=${2:-300} + + # Require a few consecutive successes: RGWs pause their HTTP frontends while reloading the + # realm after RGW configuration period changes, so a single success can land between pauses. + local want_consecutive=3 + local consecutive=0 + local deadline=$((SECONDS + timeout_seconds)) + local code + while [[ $SECONDS -lt $deadline ]]; do + code=$(curl --silent --output /dev/null --max-time 5 --write-out '%{http_code}' "http://${endpoint_ip}:80/") || code="000" + if [[ "$code" == "200" ]]; then + consecutive=$((consecutive + 1)) + if [[ $consecutive -ge $want_consecutive ]]; then + echo "RGW at ${endpoint_ip} answered ${consecutive} consecutive requests" + return 0 + fi + else + echo "RGW at ${endpoint_ip} is not serving yet (HTTP code ${code})" + consecutive=0 + fi + sleep 2 + done + echo "timed out after ${timeout_seconds}s waiting for RGW at ${endpoint_ip} to serve HTTP" + return 1 +} + +function wait_for_sync_status() { + local ns=${1?namespace is required} + local zone=${2?zone name is required} + local want=${3?expected sync status line is required} + local timeout_seconds=${4:-600} + + local deadline=$((SECONDS + timeout_seconds)) + local status="" + while [[ $SECONDS -lt $deadline ]]; do + status=$(kubectl -n "$ns" exec deploy/rook-ceph-tools -- \ + radosgw-admin sync status --rgw-realm=realm-a --rgw-zonegroup=zonegroup-a --rgw-zone="$zone") || status="" + if grep -q "$want" <<<"$status"; then + echo "zone ${zone} reports \"${want}\"" + return 0 + fi + echo "waiting for zone ${zone} to report \"${want}\"" + sleep 10 + done + echo "timed out after ${timeout_seconds}s waiting for zone ${zone} to report \"${want}\"; last sync status:" + echo "$status" + return 1 +} + +function wait_for_multisite_sync_established() { + # Wait until both zones report multisite sync fully established before exercising + # replication, mirroring the checkpoints ceph's own multisite QA performs before asserting + # anything. This also rides out the RGW frontend pauses caused by the configuration period + # changes that follow the second zone joining the zonegroup. + wait_for_sync_status rook-ceph-secondary zone-b "metadata is caught up with master" 600 + wait_for_sync_status rook-ceph-secondary zone-b "data is caught up with source" 600 + wait_for_sync_status rook-ceph zone-a "data is caught up with source" 600 +} + +function dump_multisite_diagnostics() { + set +e + local ns + for ns in rook-ceph rook-ceph-secondary; do + echo "===== ${ns}: pods" + kubectl -n "$ns" get pods -o wide + echo "===== ${ns}: rgw pod details" + kubectl -n "$ns" describe pods -l app=rook-ceph-rgw + echo "===== ${ns}: rgw pod logs" + kubectl -n "$ns" logs -l app=rook-ceph-rgw --all-containers --tail=100 + echo "===== ${ns}: sync status" + kubectl -n "$ns" exec deploy/rook-ceph-tools -- radosgw-admin sync status + echo "===== ${ns}: period" + kubectl -n "$ns" exec deploy/rook-ceph-tools -- radosgw-admin period get + done + set -e + return 0 +} + function write_object_read_from_replica_cluster() { local write_cluster_ip=${1?ip address of cluster to write to is required} local read_cluster_ip=${2?ip address of cluster to read from is required} local test_bucket_name=${3?name of the test bucket is required} + local write_zone=${4?name of the zone being written to is required} + local read_zone=${5?name of the zone being read from is required} + local read_cluster_ns=${6?namespace of the cluster being read from is required} local test_object_name="${test_bucket_name}-1mib-test.dat" fallocate -l 1M "$test_object_name" @@ -651,29 +749,22 @@ function write_object_read_from_replica_cluster() { # ensure that test file has unique data echo "$test_object_name" >>"$test_object_name" - retry 3 s3cmd --host="${write_cluster_ip}" mb "s3://${test_bucket_name}" - retry 3 s3cmd --host="${write_cluster_ip}" put "$test_object_name" "s3://${test_bucket_name}" + # RGWs pause their HTTP frontends to reload the realm whenever the RGW configuration period + # changes, and period changes keep happening for a while after the second zone joins the + # zonegroup. While a frontend is paused, connections are refused, so give the writes a time + # budget rather than a fixed number of quick retries. + retry_for 120 s3cmd --host="${write_cluster_ip}" mb "s3://${test_bucket_name}" + retry_for 120 s3cmd --host="${write_cluster_ip}" put "$test_object_name" "s3://${test_bucket_name}" - # Schedule a signal for 60s into the future as a timeout on retrying s3cmd. - # This voodoo is to avoid running everything under a new shell started by - # `timeout`, as there would be no way to pass functions to as it wouldn't be - # a direct sub-shell. - S3CMD_ERROR=0 - ( - sleep 300 - kill -s SIGUSR1 $$ - ) 2>/dev/null & - trap "{ S3CMD_ERROR=1; break; }" SIGUSR1 + # Wait until the reading zone's sync position for the bucket catches up with the source + # zone's position, the same marker-based barrier ceph's own multisite QA uses. The bucket + # metadata has to arrive on the reading zone via metadata sync before the checkpoint can + # resolve the bucket at all, hence the outer retry. + retry_for 120 kubectl -n "$read_cluster_ns" exec deploy/rook-ceph-tools -- \ + radosgw-admin bucket sync checkpoint --rgw-realm=realm-a --rgw-zonegroup=zonegroup-a --rgw-zone="$read_zone" \ + --bucket="$test_bucket_name" --source-zone="$write_zone" --retry-delay-ms=5000 --timeout-sec=300 - until s3cmd --host="${read_cluster_ip}" get "s3://${test_bucket_name}/${test_object_name}" "${test_object_name}.get" --force; do - echo "waiting for object to be replicated" - sleep 5 - done - - if [[ $S3CMD_ERROR != 0 ]]; then - echo "s3cmd failed" - exit $S3CMD_ERROR - fi + retry_for 60 s3cmd --host="${read_cluster_ip}" get "s3://${test_bucket_name}/${test_object_name}" "${test_object_name}.get" --force diff "$test_object_name" "${test_object_name}.get" } @@ -696,8 +787,12 @@ function test_multisite_object_replication() { use_https = False EOF - write_object_read_from_replica_cluster "$cluster_1_ip" "$cluster_2_ip" test1 - write_object_read_from_replica_cluster "$cluster_2_ip" "$cluster_1_ip" test2 + # both RGWs must be reliably serving before anything is written + wait_for_rgw_http "$cluster_1_ip" + wait_for_rgw_http "$cluster_2_ip" + + write_object_read_from_replica_cluster "$cluster_1_ip" "$cluster_2_ip" test1 zone-a zone-b rook-ceph-secondary + write_object_read_from_replica_cluster "$cluster_2_ip" "$cluster_1_ip" test2 zone-b zone-a rook-ceph } function create_helm_tag() {