test: update multus canary test

Update the multus canary test to reflect modern knowledge about how it
should be configured.

No longer test for the network device in OSD pods. Pods will utterly
fail to start if Multus is unable to attach interfaces.
Instead, look to the OSD map to test the connections more wholistically.
OSDs must have map IPs that include both public and cluster network.
This implicitly tests that the interfaces exist in the Pod, and it
additionally verifies other details, like Ceph `*_network` configs are
set propertly.

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>

.github/workflows/canary-integration-test.yml

@@ -1534,7 +1534,7 @@ jobs:
         run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
 
       - name: wait for ceph to be ready
-        run: IS_POD_NETWORK=true IS_MULTUS=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd_multus 2
+        run: IS_POD_NETWORK=true IS_MULTUS=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
 
       - name: wait for ceph-csi configmap to be updated with network namespace
         run: tests/scripts/github-action-helper.sh wait_for_ceph_csi_configmap_to_be_updated
@@ -1542,6 +1542,9 @@ jobs:
       - name: wait for cephnfs to be ready
         run: IS_POD_NETWORK=true IS_MULTUS=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready nfs 1
 
+      - name: check multus connections
+        run: tests/scripts/github-action-helper.sh test_multus_connections
+
       - name: test ceph-csi-rbd plugin restart
         run: tests/scripts/github-action-helper.sh test_csi_rbd_workload
 

tests/scripts/github-action-helper.sh

@@ -603,6 +603,14 @@ function deploy_multus_cluster() {
   kubectl create -f nfs-test.yaml
 }
 
+function test_multus_connections() {
+  EXEC='kubectl -n rook-ceph exec -t deploy/rook-ceph-tools -- ceph --connect-timeout 10'
+  # each OSD should exist on both public and cluster network
+  $EXEC osd dump | grep osd.0 | grep "192.168.20." | grep "192.168.21."
+  # MDSes should exist on public network and NOT on cluster network
+  $EXEC fs dump | grep myfs-a | grep "192.168.20." | grep -v "192.168.21."
+}
+
 function create_operator_toolbox() {
   cd deploy/examples
   sed -i "s|image: rook/ceph:.*|image: rook/ceph:local-build|g" toolbox-operator-image.yaml

tests/scripts/validate_cluster.sh

@@ -27,7 +27,7 @@ OSD_COUNT=$2
 #############
 EXEC_COMMAND="kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') -- ceph --connect-timeout 10"
 
-function wait_for_daemon () {
+function wait_for_daemon() {
   timeout=90
   daemon_to_test=$1
   while [ $timeout -ne 0 ]; do
@@ -62,7 +62,7 @@ function test_demo_osd {
 }
 
 function test_demo_rgw {
-    timeout 360 bash -x <<-'EOF'
+  timeout 360 bash -x <<-'EOF'
     until [[ "$(kubectl -n rook-ceph get pods -l app=rook-ceph-rgw -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" == "True" ]]; do
       echo "waiting for rgw pods to be ready"
       sleep 5
@@ -86,7 +86,7 @@ function test_demo_rbd_mirror {
 
 function test_demo_fs_mirror {
   # shellcheck disable=SC2046
-    return $(wait_for_daemon "$EXEC_COMMAND -s | grep -sq 'cephfs-mirror:'")
+  return $(wait_for_daemon "$EXEC_COMMAND -s | grep -sq 'cephfs-mirror:'")
 }
 
 function test_demo_pool {
@@ -127,13 +127,6 @@ function test_nfs {
 EOF
 }
 
-function test_multus_osd {
-  for i in $(seq 1 2); do
-    kubectl -n rook-ceph exec -t deploy/rook-ceph-osd-0 -c osd -- grep net"$i" /proc/net/dev
-    kubectl -n rook-ceph exec -t deploy/rook-ceph-osd-0 -c osd -- grep net"$i" /proc/net/dev
-  done
-}
-
 ########
 # MAIN #
 ########
@@ -148,7 +141,7 @@ else
   comma_to_space=${DAEMON_TO_VALIDATE//,/ }
 
   # transform to an array
-  IFS=" " read -r -a array <<< "$comma_to_space"
+  IFS=" " read -r -a array <<<"$comma_to_space"
 
   # sort and remove potential duplicate
   daemons_list=$(echo "${array[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')
@@ -156,39 +149,35 @@ fi
 
 for daemon in $daemons_list; do
   case "$daemon" in
-    mon)
-      continue
-      ;;
-    mgr)
-      continue
-      ;;
-    osd)
-      test_demo_osd
-      ;;
-    osd_multus)
-      test_demo_osd
-      test_multus_osd
-      ;;
-    mds)
-      test_demo_mds
-      ;;
-    rgw)
-      test_demo_rgw
-      ;;
-    rbd_mirror)
-      test_demo_rbd_mirror
-      ;;
-    fs_mirror)
-      test_demo_fs_mirror
-      ;;
-    nfs)
-      test_nfs
-      ;;
-    *)
-      log "ERROR: unknown daemon to validate!"
-      log "Available daemon are: mon mgr osd mds rgw rbd_mirror fs_mirror"
-      exit 1
-      ;;
+  mon)
+    continue
+    ;;
+  mgr)
+    continue
+    ;;
+  osd)
+    test_demo_osd
+    ;;
+  mds)
+    test_demo_mds
+    ;;
+  rgw)
+    test_demo_rgw
+    ;;
+  rbd_mirror)
+    test_demo_rbd_mirror
+    ;;
+  fs_mirror)
+    test_demo_fs_mirror
+    ;;
+  nfs)
+    test_nfs
+    ;;
+  *)
+    log "ERROR: unknown daemon to validate!"
+    log "Available daemon are: mon mgr osd mds rgw rbd_mirror fs_mirror"
+    exit 1
+    ;;
   esac
 done