From 4da654bae2ff1fb8da56fd4c77b6681a3a994d24 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Wed, 10 Jun 2026 16:54:24 -0700 Subject: [PATCH] ci: fix multus integration test setup flakes Multus CI failures on master pushes from 2026-04-08 to 2026-06-10 were dominated by a race in setup-multus.sh: 'kubectl wait' was invoked on a pod label selector immediately after 'kubectl create' of a daemonset, and it exits with "error: no matching resources found" when the daemonset controller has not created any pods yet. This caused 9 of the 10 genuine multus flakes across the standalone multus workflow and the canary multus-public-and-cluster job (the two share setup-multus.sh). - setup-multus.sh: wait with 'kubectl rollout status' on the daemonsets instead of 'kubectl wait' on pod label selectors. The daemonset object exists as soon as 'kubectl create' returns, and rollout status correctly waits for all desired pods to be created and become ready. The old selector wait also silently under-waited: pods created after its initial LIST were never waited on at all. The stricter wait revealed that full multi-node convergence can exceed 2 minutes on busy runners, so the wait timeout is raised to 5 minutes (the waits return as soon as the rollouts are ready, so this costs nothing on healthy runs). - Wait for the host-net-config daemonset rollout in both workflows before proceeding; it configures the host routing that the multus public network depends on and was previously not waited on at all. Pin its jonlabelle/network-tools image. - test_multus_connections: retry the osd dump / fs dump network checks for up to 2 minutes. Daemons register their addresses in the mon maps asynchronously after the cluster reports ready; one canary failure (2026-04-29) ran the MDS check at fsmap epoch 1, before any MDS had registered. - Dump cluster state (pods, daemonsets, events, multus logs) when the multus workflow job fails; the workflow previously had no failure diagnostics at all. Also remove the unused NUMBER_OF_COMPUTE_NODES env var from the multus workflow (kind-config.yaml creates 3 workers; nothing consumes it). Signed-off-by: Joshua Hoblitt --- .github/workflows/canary-integration-test.yml | 4 +++- .github/workflows/multus.yaml | 15 ++++++++++++--- tests/scripts/github-action-helper.sh | 7 +++++-- tests/scripts/multus/host-cfg-ds.yaml | 4 +++- tests/scripts/multus/setup-multus.sh | 14 ++++++++++---- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 343404a00..d2fb4be0c 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -2110,7 +2110,9 @@ jobs: run: ./tests/scripts/multus/setup-multus.sh - name: Set up multus prerequisite host routing - run: kubectl create -f tests/scripts/multus/host-cfg-ds.yaml + run: | + kubectl create -f tests/scripts/multus/host-cfg-ds.yaml + kubectl rollout status daemonset/host-net-config --timeout=120s - name: create cluster prerequisites run: tests/scripts/github-action-helper.sh create_cluster_prerequisites diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml index 18dd73573..37d7a87aa 100644 --- a/.github/workflows/multus.yaml +++ b/.github/workflows/multus.yaml @@ -32,8 +32,6 @@ jobs: strategy: fail-fast: true if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - env: - NUMBER_OF_COMPUTE_NODES: 5 steps: - name: checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -61,7 +59,9 @@ jobs: run: ./tests/scripts/multus/setup-multus.sh - name: Set up multus prerequisite host routing - run: kubectl create -f tests/scripts/multus/host-cfg-ds.yaml + run: | + kubectl create -f tests/scripts/multus/host-cfg-ds.yaml + kubectl rollout status daemonset/host-net-config --timeout=120s - name: Install public and cluster NADs in default namespace run: kubectl create -f tests/scripts/multus/default-public-cluster-nads.yaml @@ -112,6 +112,15 @@ jobs: if: steps.taint.outcome == 'success' && !cancelled() run: ./tests/scripts/multus/test-240-stretch-cluster-only.sh + - name: dump cluster state on failure + if: failure() + run: | + kubectl get nodes -o wide || true + kubectl get pods -A -o wide || true + kubectl -n kube-system describe daemonset kube-multus-ds install-cni-plugins whereabouts || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl -n kube-system logs -l name=multus --all-containers --tail=30 || true + - name: consider (post-job) debugging uses: ./.github/workflows/upterm_debug with: diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 5ee3fe905..e98bcebae 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -708,10 +708,13 @@ function create_helm_tag() { function test_multus_connections() { EXEC='kubectl -n rook-ceph exec -t deploy/rook-ceph-tools -- ceph --connect-timeout 10' + # daemons register their network addresses in the mon maps asynchronously + # after the cluster reports ready (e.g., MDSes may not be in the fsmap yet), + # so retry each check until it converges # each OSD should exist on both public and cluster network - $EXEC osd dump | grep osd.0 | grep "192.168.20." | grep "192.168.21." + timeout 120 bash -c "until $EXEC osd dump | grep osd.0 | grep '192.168.20.' | grep '192.168.21.'; do sleep 5; done" # MDSes should exist on public network and NOT on cluster network - $EXEC fs dump | grep myfs-a | grep "192.168.20." | grep -v "192.168.21." + timeout 120 bash -c "until $EXEC fs dump | grep myfs-a | grep '192.168.20.' | grep -v '192.168.21.'; do sleep 5; done" } function create_operator_toolbox() { diff --git a/tests/scripts/multus/host-cfg-ds.yaml b/tests/scripts/multus/host-cfg-ds.yaml index e3e868617..747ee4857 100644 --- a/tests/scripts/multus/host-cfg-ds.yaml +++ b/tests/scripts/multus/host-cfg-ds.yaml @@ -29,7 +29,9 @@ spec: terminationGracePeriodSeconds: 0 # allow updating/deleting immediately containers: - name: test - image: jonlabelle/network-tools + # tag sha-6257a44 == the image's 'latest' as of 2026-06-10, pinned + # so the moving tag cannot break or slow CI + image: jonlabelle/network-tools:sha-6257a44 env: - name: IFACE_NAME value: eth0 # IFACE_NAME diff --git a/tests/scripts/multus/setup-multus.sh b/tests/scripts/multus/setup-multus.sh index cadc0287b..b3d7edcd0 100755 --- a/tests/scripts/multus/setup-multus.sh +++ b/tests/scripts/multus/setup-multus.sh @@ -7,7 +7,9 @@ MULTUS_DAEMONSET_URL="https://raw.githubusercontent.com/k8snetworkplumbingwg/mul RETRY_MAX=10 INTERVAL=10 TIMEOUT=60 -TIMEOUT_K8="120s" +# image pulls plus CNI bootstrap across all nodes can take a few minutes on +# busy CI runners; the waits below return as soon as the rollouts are ready +TIMEOUT_K8="300s" retry() { local status=0 @@ -36,17 +38,21 @@ kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=$ echo "## install multus" retry kubectl create -f "${MULTUS_DAEMONSET_URL}" -kubectl -n kube-system wait --for=condition=ready -l name="multus" pod --timeout=$TIMEOUT_K8 +# use 'rollout status' on the daemonset instead of 'kubectl wait' on a pod +# label selector: right after 'kubectl create' the daemonset controller may +# not have created any pods yet, and 'kubectl wait' exits immediately with +# "no matching resources found" when the selector matches nothing +kubectl -n kube-system rollout status daemonset/kube-multus-ds --timeout=$TIMEOUT_K8 echo "## install CNIs" retry kubectl create -f "https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/hack/cni-install.yml" -kubectl -n kube-system wait --for=condition=ready -l name="cni-plugins" pod --timeout=$TIMEOUT_K8 +kubectl -n kube-system rollout status daemonset/install-cni-plugins --timeout=$TIMEOUT_K8 echo "## install whereabouts" kubectl create \ -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/daemonset-install.yaml \ -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml \ -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml -kubectl -n kube-system wait --for=condition=ready -l app=whereabouts pod --timeout=$TIMEOUT_K8 +kubectl -n kube-system rollout status daemonset/whereabouts --timeout=$TIMEOUT_K8 echo "#### set up multus done ####"