diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 343404a00..d2fb4be0c 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -2110,7 +2110,9 @@ jobs: run: ./tests/scripts/multus/setup-multus.sh - name: Set up multus prerequisite host routing - run: kubectl create -f tests/scripts/multus/host-cfg-ds.yaml + run: | + kubectl create -f tests/scripts/multus/host-cfg-ds.yaml + kubectl rollout status daemonset/host-net-config --timeout=120s - name: create cluster prerequisites run: tests/scripts/github-action-helper.sh create_cluster_prerequisites diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml index 18dd73573..37d7a87aa 100644 --- a/.github/workflows/multus.yaml +++ b/.github/workflows/multus.yaml @@ -32,8 +32,6 @@ jobs: strategy: fail-fast: true if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - env: - NUMBER_OF_COMPUTE_NODES: 5 steps: - name: checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -61,7 +59,9 @@ jobs: run: ./tests/scripts/multus/setup-multus.sh - name: Set up multus prerequisite host routing - run: kubectl create -f tests/scripts/multus/host-cfg-ds.yaml + run: | + kubectl create -f tests/scripts/multus/host-cfg-ds.yaml + kubectl rollout status daemonset/host-net-config --timeout=120s - name: Install public and cluster NADs in default namespace run: kubectl create -f tests/scripts/multus/default-public-cluster-nads.yaml @@ -112,6 +112,15 @@ jobs: if: steps.taint.outcome == 'success' && !cancelled() run: ./tests/scripts/multus/test-240-stretch-cluster-only.sh + - name: dump cluster state on failure + if: failure() + run: | + kubectl get nodes -o wide || true + kubectl get pods -A -o wide || true + kubectl -n kube-system describe daemonset kube-multus-ds install-cni-plugins whereabouts || true + kubectl get events -A --sort-by=.lastTimestamp | tail -60 || true + kubectl -n kube-system logs -l name=multus --all-containers --tail=30 || true + - name: consider (post-job) debugging uses: ./.github/workflows/upterm_debug with: diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 5ee3fe905..e98bcebae 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -708,10 +708,13 @@ function create_helm_tag() { function test_multus_connections() { EXEC='kubectl -n rook-ceph exec -t deploy/rook-ceph-tools -- ceph --connect-timeout 10' + # daemons register their network addresses in the mon maps asynchronously + # after the cluster reports ready (e.g., MDSes may not be in the fsmap yet), + # so retry each check until it converges # each OSD should exist on both public and cluster network - $EXEC osd dump | grep osd.0 | grep "192.168.20." | grep "192.168.21." + timeout 120 bash -c "until $EXEC osd dump | grep osd.0 | grep '192.168.20.' | grep '192.168.21.'; do sleep 5; done" # MDSes should exist on public network and NOT on cluster network - $EXEC fs dump | grep myfs-a | grep "192.168.20." | grep -v "192.168.21." + timeout 120 bash -c "until $EXEC fs dump | grep myfs-a | grep '192.168.20.' | grep -v '192.168.21.'; do sleep 5; done" } function create_operator_toolbox() { diff --git a/tests/scripts/multus/host-cfg-ds.yaml b/tests/scripts/multus/host-cfg-ds.yaml index e3e868617..747ee4857 100644 --- a/tests/scripts/multus/host-cfg-ds.yaml +++ b/tests/scripts/multus/host-cfg-ds.yaml @@ -29,7 +29,9 @@ spec: terminationGracePeriodSeconds: 0 # allow updating/deleting immediately containers: - name: test - image: jonlabelle/network-tools + # tag sha-6257a44 == the image's 'latest' as of 2026-06-10, pinned + # so the moving tag cannot break or slow CI + image: jonlabelle/network-tools:sha-6257a44 env: - name: IFACE_NAME value: eth0 # IFACE_NAME diff --git a/tests/scripts/multus/setup-multus.sh b/tests/scripts/multus/setup-multus.sh index cadc0287b..b3d7edcd0 100755 --- a/tests/scripts/multus/setup-multus.sh +++ b/tests/scripts/multus/setup-multus.sh @@ -7,7 +7,9 @@ MULTUS_DAEMONSET_URL="https://raw.githubusercontent.com/k8snetworkplumbingwg/mul RETRY_MAX=10 INTERVAL=10 TIMEOUT=60 -TIMEOUT_K8="120s" +# image pulls plus CNI bootstrap across all nodes can take a few minutes on +# busy CI runners; the waits below return as soon as the rollouts are ready +TIMEOUT_K8="300s" retry() { local status=0 @@ -36,17 +38,21 @@ kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=$ echo "## install multus" retry kubectl create -f "${MULTUS_DAEMONSET_URL}" -kubectl -n kube-system wait --for=condition=ready -l name="multus" pod --timeout=$TIMEOUT_K8 +# use 'rollout status' on the daemonset instead of 'kubectl wait' on a pod +# label selector: right after 'kubectl create' the daemonset controller may +# not have created any pods yet, and 'kubectl wait' exits immediately with +# "no matching resources found" when the selector matches nothing +kubectl -n kube-system rollout status daemonset/kube-multus-ds --timeout=$TIMEOUT_K8 echo "## install CNIs" retry kubectl create -f "https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/hack/cni-install.yml" -kubectl -n kube-system wait --for=condition=ready -l name="cni-plugins" pod --timeout=$TIMEOUT_K8 +kubectl -n kube-system rollout status daemonset/install-cni-plugins --timeout=$TIMEOUT_K8 echo "## install whereabouts" kubectl create \ -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/daemonset-install.yaml \ -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml \ -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml -kubectl -n kube-system wait --for=condition=ready -l app=whereabouts pod --timeout=$TIMEOUT_K8 +kubectl -n kube-system rollout status daemonset/whereabouts --timeout=$TIMEOUT_K8 echo "#### set up multus done ####"