From 3e2e906ea750be69f26dc28baf13f3f2c16bbe21 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Mon, 18 Sep 2023 15:03:10 -0600 Subject: [PATCH] test: update multus canary test Update the multus canary test to reflect modern knowledge about how it should be configured. No longer test for the network device in OSD pods. Pods will utterly fail to start if Multus is unable to attach interfaces. Instead, look to the OSD map to test the connections more wholistically. OSDs must have map IPs that include both public and cluster network. This implicitly tests that the interfaces exist in the Pod, and it additionally verifies other details, like Ceph `*_network` configs are set propertly. Signed-off-by: Blaine Gardner --- .github/workflows/canary-integration-test.yml | 5 +- tests/scripts/github-action-helper.sh | 8 ++ tests/scripts/validate_cluster.sh | 77 ++++++++----------- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 6d88f4072..08ac9057a 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1534,7 +1534,7 @@ jobs: run: tests/scripts/github-action-helper.sh wait_for_prepare_pod - name: wait for ceph to be ready - run: IS_POD_NETWORK=true IS_MULTUS=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd_multus 2 + run: IS_POD_NETWORK=true IS_MULTUS=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 - name: wait for ceph-csi configmap to be updated with network namespace run: tests/scripts/github-action-helper.sh wait_for_ceph_csi_configmap_to_be_updated @@ -1542,6 +1542,9 @@ jobs: - name: wait for cephnfs to be ready run: IS_POD_NETWORK=true IS_MULTUS=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready nfs 1 + - name: check multus connections + run: tests/scripts/github-action-helper.sh test_multus_connections + - name: test ceph-csi-rbd plugin restart run: tests/scripts/github-action-helper.sh test_csi_rbd_workload diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index a921471f5..b5dbd1f45 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -603,6 +603,14 @@ function deploy_multus_cluster() { kubectl create -f nfs-test.yaml } +function test_multus_connections() { + EXEC='kubectl -n rook-ceph exec -t deploy/rook-ceph-tools -- ceph --connect-timeout 10' + # each OSD should exist on both public and cluster network + $EXEC osd dump | grep osd.0 | grep "192.168.20." | grep "192.168.21." + # MDSes should exist on public network and NOT on cluster network + $EXEC fs dump | grep myfs-a | grep "192.168.20." | grep -v "192.168.21." +} + function create_operator_toolbox() { cd deploy/examples sed -i "s|image: rook/ceph:.*|image: rook/ceph:local-build|g" toolbox-operator-image.yaml diff --git a/tests/scripts/validate_cluster.sh b/tests/scripts/validate_cluster.sh index c918ed119..306a1eece 100755 --- a/tests/scripts/validate_cluster.sh +++ b/tests/scripts/validate_cluster.sh @@ -27,7 +27,7 @@ OSD_COUNT=$2 ############# EXEC_COMMAND="kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') -- ceph --connect-timeout 10" -function wait_for_daemon () { +function wait_for_daemon() { timeout=90 daemon_to_test=$1 while [ $timeout -ne 0 ]; do @@ -62,7 +62,7 @@ function test_demo_osd { } function test_demo_rgw { - timeout 360 bash -x <<-'EOF' + timeout 360 bash -x <<-'EOF' until [[ "$(kubectl -n rook-ceph get pods -l app=rook-ceph-rgw -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" == "True" ]]; do echo "waiting for rgw pods to be ready" sleep 5 @@ -86,7 +86,7 @@ function test_demo_rbd_mirror { function test_demo_fs_mirror { # shellcheck disable=SC2046 - return $(wait_for_daemon "$EXEC_COMMAND -s | grep -sq 'cephfs-mirror:'") + return $(wait_for_daemon "$EXEC_COMMAND -s | grep -sq 'cephfs-mirror:'") } function test_demo_pool { @@ -127,13 +127,6 @@ function test_nfs { EOF } -function test_multus_osd { - for i in $(seq 1 2); do - kubectl -n rook-ceph exec -t deploy/rook-ceph-osd-0 -c osd -- grep net"$i" /proc/net/dev - kubectl -n rook-ceph exec -t deploy/rook-ceph-osd-0 -c osd -- grep net"$i" /proc/net/dev - done -} - ######## # MAIN # ######## @@ -148,7 +141,7 @@ else comma_to_space=${DAEMON_TO_VALIDATE//,/ } # transform to an array - IFS=" " read -r -a array <<< "$comma_to_space" + IFS=" " read -r -a array <<<"$comma_to_space" # sort and remove potential duplicate daemons_list=$(echo "${array[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ') @@ -156,39 +149,35 @@ fi for daemon in $daemons_list; do case "$daemon" in - mon) - continue - ;; - mgr) - continue - ;; - osd) - test_demo_osd - ;; - osd_multus) - test_demo_osd - test_multus_osd - ;; - mds) - test_demo_mds - ;; - rgw) - test_demo_rgw - ;; - rbd_mirror) - test_demo_rbd_mirror - ;; - fs_mirror) - test_demo_fs_mirror - ;; - nfs) - test_nfs - ;; - *) - log "ERROR: unknown daemon to validate!" - log "Available daemon are: mon mgr osd mds rgw rbd_mirror fs_mirror" - exit 1 - ;; + mon) + continue + ;; + mgr) + continue + ;; + osd) + test_demo_osd + ;; + mds) + test_demo_mds + ;; + rgw) + test_demo_rgw + ;; + rbd_mirror) + test_demo_rbd_mirror + ;; + fs_mirror) + test_demo_fs_mirror + ;; + nfs) + test_nfs + ;; + *) + log "ERROR: unknown daemon to validate!" + log "Available daemon are: mon mgr osd mds rgw rbd_mirror fs_mirror" + exit 1 + ;; esac done