ci: add multus integration test

This new integration test will deploy a cluster with multus enabled. It
will be comprised of two network interfaces for ceph public and cluster
communications.
For now, it only deploys a Ceph cluster up to the OSDs.

Closes: https://github.com/rook/rook/issues/9784
Signed-off-by: Sébastien Han <seb@redhat.com>

.github/workflows/canary-integration-test.yml

@@ -908,3 +908,48 @@ jobs:
       if: failure() && github.event_name == 'pull_request'
       uses: mxschmitt/action-tmate@v3
       timeout-minutes: 60
+
+  multus-cluster-network:
+    runs-on: ubuntu-18.04
+    if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
+    steps:
+    - name: checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: setup cluster resources
+      uses: ./.github/workflows/setup-cluster-resources
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: validate-yaml
+      run: tests/scripts/github-action-helper.sh validate_yaml
+
+    - name: use local disk and create partitions for osds
+      run: |
+        tests/scripts/github-action-helper.sh use_local_disk
+        tests/scripts/github-action-helper.sh create_partitions_for_osds
+
+    - name: deploy multus
+      run: tests/scripts/github-action-helper.sh deploy_multus
+
+    - name: deploy multus cluster
+      run: tests/scripts/github-action-helper.sh deploy_multus_cluster
+
+    - name: wait for prepare pod
+      run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
+
+    - name: wait for ceph to be ready
+      run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
+
+    - name: collect common logs
+      if: always()
+      uses: ./.github/workflows/collect-logs
+      with:
+        name: canary-multus
+
+    - name: setup tmate session for debugging when event is PR
+      if: failure() && github.event_name == 'pull_request'
+      uses: mxschmitt/action-tmate@v3
+      timeout-minutes: 60

.github/workflows/setup-cluster-resources/action.yaml

@@ -18,7 +18,7 @@ runs:
       with:
         minikube version: 'v1.24.0'
         kubernetes version: 'v1.23.0'
-        start args: --memory 6g --cpus=2 --addons ingress
+        start args: --memory 6g --cpus=2 --addons ingress --cni=flannel
         github token: ${{ inputs.github-token }}
 
     - name: install deps

deploy/examples/cluster-multus-test.yaml

@@ -0,0 +1,49 @@
+#################################################################################################################
+# Define the settings for the rook-ceph cluster with common settings for a small test cluster.
+# All nodes with available raw devices will be used for the Ceph cluster. One node is sufficient
+# in this example.
+
+# For example, to create the cluster:
+#   kubectl create -f crds.yaml -f common.yaml -f operator.yaml
+#   kubectl create -f cluster-test.yaml
+#################################################################################################################
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: rook-config-override
+  namespace: rook-ceph # namespace:cluster
+data:
+  config: |
+    [global]
+    osd_pool_default_size = 1
+    mon_warn_on_pool_no_redundancy = false
+    bdev_flock_retry = 20
+    bluefs_buffered_io = false
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: my-cluster
+  namespace: rook-ceph # namespace:cluster
+spec:
+  dataDirHostPath: /var/lib/rook
+  cephVersion:
+    image: quay.io/ceph/ceph:v17
+    allowUnsupported: true
+  mon:
+    count: 1
+  mgr:
+    count: 1
+  dashboard:
+    enabled: true
+  network:
+    provider: multus
+    selectors:
+      public: public-net
+      cluster: cluster-net
+  crashCollector:
+    disable: true
+  storage:
+    useAllNodes: true
+    useAllDevices: true
+    #deviceFilter:

tests/scripts/github-action-helper.sh

@@ -379,6 +379,56 @@ function create_helm_tag() {
   docker tag "${build_image}" "rook/ceph:${helm_tag}"
 }
 
+function deploy_multus() {
+  # download the multus daemonset, and remove mem and cpu limits that cause it to crash on minikube
+  curl https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick-plugin.yml \
+    | sed -e 's/cpu: /# cpu: /g' -e 's/memory: /# memory: /g' \
+    | kubectl apply -f -
+
+  # install whereabouts
+  kubectl apply \
+    -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/daemonset-install.yaml \
+    -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/ip-reconciler-job.yaml \
+    -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml \
+    -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml
+
+  # create the rook-ceph namespace if it doesn't exist, the NAD will go in this namespace
+  kubectl create namespace rook-ceph || true
+
+  # install network attachment definitions
+  IFACE="eth0" # the runner has eth0 so we don't need any heureustics to find the interface
+  kubectl apply -f - <<EOF
+---
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: public-net
+  namespace: rook-ceph
+  labels:
+  annotations:
+spec:
+  config: '{ "cniVersion": "0.3.0", "type": "macvlan", "master": "$IFACE", "mode": "bridge", "ipam": { "type": "whereabouts", "range": "192.168.20.0/24" } }'
+---
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: cluster-net
+  namespace: rook-ceph
+  labels:
+  annotations:
+spec:
+  config: '{ "cniVersion": "0.3.0", "type": "macvlan", "master": "$IFACE", "mode": "bridge", "ipam": { "type": "whereabouts", "range": "192.168.21.0/24" } }'
+EOF
+}
+
+function deploy_multus_cluster() {
+  cd deploy/examples
+  deploy_manifest_with_local_build operator.yaml
+  deploy_manifest_with_local_build toolbox.yaml
+  sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}|g" cluster-multus-test.yaml
+  kubectl create -f cluster-multus-test.yaml
+}
+
 FUNCTION="$1"
 shift # remove function arg now that we've recorded it
 # call the function with the remainder of the user-provided args

tests/scripts/validate_cluster.sh

@@ -89,7 +89,7 @@ function test_demo_pool {
 
 function test_csi {
   timeout 360 bash <<-'EOF'
-    until [[ "$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running|grep -c ^csi-)" -eq 4 ]]; do
+    until [[ "$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running|grep -c ^csi-)" -ge 4 ]]; do
       echo "waiting for csi pods to be ready"
       sleep 5
     done