diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml new file mode 100644 index 000000000..ceac62a6f --- /dev/null +++ b/.github/workflows/multus.yaml @@ -0,0 +1,76 @@ +name: Multus integration tests +on: + push: + tags: + - v* + branches: + - master + - release-* + pull_request: + paths: + - cmd/userfacing/** + - pkg/daemon/multus/** + - .github/workflows/multus.yaml + +defaults: + run: + # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell + shell: bash --noprofile --norc -eo pipefail -x {0} + +# cancel the in-progress workflow when PR is refreshed. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +jobs: + test-validation-tool: + runs-on: ubuntu-latest + strategy: + fail-fast: true + if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" + env: + NUMBER_OF_COMPUTE_NODES: 5 + steps: + - name: Set up Go version + uses: actions/setup-go@v1 + with: + go-version: 1.19.x + + - name: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Create KinD Cluster + uses: helm/kind-action@v1.7.0 + with: + config: tests/scripts/multus/kind-config.yaml + cluster_name: rook-multus-e2e + + - name: Start tmate + if: runner.debug || contains(github.event.pull_request.labels.*.name, 'debug-ci') + run: | + kubectl apply -f tests/scripts/tmate-pod.yaml + sleep 3 # sometimes the next command errors b/c k8s hasn't had time to schedule the pod yet + kubectl -n tmate wait --for=condition=ready -l app=tmate pod --timeout=300s + sleep 1 # just in case tmate hasn't output its web/ssh links yet + kubectl -n tmate logs deploy/tmate + + - name: Setup multus + run: ./tests/scripts/multus/setup-multus.sh + + - name: Install public and cluster NADs in default namespace + run: kubectl create -f tests/scripts/multus/default-public-cluster-nads.yaml + + - name: Quickly build Rook binary + run: go build -o rook cmd/rook/*.go + + - name: Run multus validation test + run: | + export KUBECONFIG="/home/runner/.kube/config" + kubectl create namespace rook-ceph + ./rook --log-level debug multus validation run \ + --namespace rook-ceph \ + --public-network default/public-net \ + --cluster-network default/cluster-net \ + --daemons-per-node 2 diff --git a/tests/scripts/multus/default-public-cluster-nads.yaml b/tests/scripts/multus/default-public-cluster-nads.yaml new file mode 100644 index 000000000..bfb5898b5 --- /dev/null +++ b/tests/scripts/multus/default-public-cluster-nads.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: public-net + namespace: default + labels: + annotations: +spec: + config: '{ "cniVersion": "0.3.1", "type": "macvlan", "master": "eth0", "mode": "bridge", "ipam": { "type": "whereabouts", "range": "192.168.20.0/24" } }' +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: cluster-net + namespace: default + labels: + annotations: +spec: + config: '{ "cniVersion": "0.3.1", "type": "macvlan", "master": "eth0", "mode": "bridge", "ipam": { "type": "whereabouts", "range": "192.168.30.0/24" } }' diff --git a/tests/scripts/multus/kind-config.yaml b/tests/scripts/multus/kind-config.yaml new file mode 100644 index 000000000..d33153a58 --- /dev/null +++ b/tests/scripts/multus/kind-config.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: rook-multus-e2e +nodes: + - role: control-plane + - role: worker + - role: worker + - role: worker diff --git a/tests/scripts/multus/setup-multus.sh b/tests/scripts/multus/setup-multus.sh new file mode 100755 index 000000000..068a7e844 --- /dev/null +++ b/tests/scripts/multus/setup-multus.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -xeo pipefail + +# initially copied from https://github.com/k8snetworkplumbingwg/whereabouts + +MULTUS_DAEMONSET_URL="https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml" +RETRY_MAX=10 +INTERVAL=10 +TIMEOUT=60 +TIMEOUT_K8="120s" + +retry() { + local status=0 + local retries=${RETRY_MAX:=5} + local delay=${INTERVAL:=5} + local to=${TIMEOUT:=20} + cmd="$*" + + while [ $retries -gt 0 ]; do + status=0 + timeout $to bash -c "echo $cmd && $cmd" || status=$? + if [ $status -eq 0 ]; then + break + fi + echo "Exit code: '$status'. Sleeping '$delay' seconds before retrying" + sleep $delay + let retries-- + done + return $status +} + +echo "#### set up multus ####" + +echo " ## wait for coreDNS" +kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=$TIMEOUT_K8 + +echo "## install multus" +retry kubectl create -f "${MULTUS_DAEMONSET_URL}" +kubectl -n kube-system wait --for=condition=ready -l name="multus" pod --timeout=$TIMEOUT_K8 + +echo "## install CNIs" +retry kubectl create -f "https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/hack/cni-install.yml" +kubectl -n kube-system wait --for=condition=ready -l name="cni-plugins" pod --timeout=$TIMEOUT_K8 + +echo "## install whereabouts" +kubectl create \ + -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/daemonset-install.yaml \ + -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml \ + -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml +kubectl -n kube-system wait --for=condition=ready -l app=whereabouts pod --timeout=$TIMEOUT_K8 + +echo "#### set up multus done ####"