From 6a3cad9a2d28f9a67de73b165abbf084048543a0 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Tue, 1 Aug 2023 17:46:14 +0530 Subject: [PATCH 1/3] ci: add github action to free space in runner adding github action to free space in runner which will help us avoiding mon low space error Signed-off-by: subhamkrai (cherry picked from commit 334a632ecd34c7bab474b06360a7639aa8f17c2f) --- .github/workflows/canary-integration-test.yml | 141 +++++++++++++++++- .../integration-test-helm-suite.yaml | 7 + .../workflows/integration-test-mgr-suite.yaml | 7 + .../integration-test-multi-cluster-suite.yaml | 7 + .../integration-test-object-suite.yaml | 7 + .../integration-test-smoke-suite.yaml | 7 + .../integration-test-upgrade-suite.yaml | 16 +- 7 files changed, 182 insertions(+), 10 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index ceb585318..cb834f9d1 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -39,6 +39,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -242,7 +249,6 @@ jobs: with: name: canary - raw-disk: runs-on: ubuntu-20.04 if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" @@ -257,6 +263,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -309,6 +322,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -356,6 +376,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -409,6 +436,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -455,6 +489,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -507,6 +548,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -580,6 +628,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -629,6 +684,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -681,6 +743,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -745,6 +814,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -796,6 +872,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -857,6 +940,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -939,6 +1029,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -1001,6 +1098,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -1055,6 +1159,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: @@ -1311,6 +1422,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: run RGW multisite test uses: ./.github/workflows/rgw-multisite-test with: @@ -1338,6 +1456,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: run encryption KMS IBM Key Protect uses: ./.github/workflows/encryption-pvc-kms-ibm-kp if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''" @@ -1372,6 +1497,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup golang uses: actions/setup-go@v4 with: @@ -1451,6 +1583,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/canary-test-config with: diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml index 115aa5f13..8496b94b7 100644 --- a/.github/workflows/integration-test-helm-suite.yaml +++ b/.github/workflows/integration-test-helm-suite.yaml @@ -37,6 +37,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup latest cluster resources uses: ./.github/workflows/integration-test-config-latest-k8s with: diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml index 8f75eeb3f..ed76893e3 100644 --- a/.github/workflows/integration-test-mgr-suite.yaml +++ b/.github/workflows/integration-test-mgr-suite.yaml @@ -36,6 +36,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/integration-test-config-latest-k8s with: diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml index 9f2a44a86..1573089ca 100644 --- a/.github/workflows/integration-test-multi-cluster-suite.yaml +++ b/.github/workflows/integration-test-multi-cluster-suite.yaml @@ -37,6 +37,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/integration-test-config-latest-k8s with: diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml index a8bf584cc..b4ca6aab7 100644 --- a/.github/workflows/integration-test-object-suite.yaml +++ b/.github/workflows/integration-test-object-suite.yaml @@ -37,6 +37,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup latest cluster resources uses: ./.github/workflows/integration-test-config-latest-k8s with: diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml index 632a8713c..bab77f764 100644 --- a/.github/workflows/integration-test-smoke-suite.yaml +++ b/.github/workflows/integration-test-smoke-suite.yaml @@ -37,6 +37,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/integration-test-config-latest-k8s with: diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml index 0669de4ec..6d072a382 100644 --- a/.github/workflows/integration-test-upgrade-suite.yaml +++ b/.github/workflows/integration-test-upgrade-suite.yaml @@ -44,15 +44,6 @@ jobs: # if set to "true" but frees about 6 GB tool-cache: true - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true - - name: setup cluster resources uses: ./.github/workflows/integration-test-config-latest-k8s with: @@ -98,6 +89,13 @@ jobs: with: use-tmate: ${{ secrets.USE_TMATE }} + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + - name: setup cluster resources uses: ./.github/workflows/integration-test-config-latest-k8s with: From fa78cd72ba81ba2ae2e7897523f517e36a434792 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Wed, 2 Aug 2023 12:35:29 +0530 Subject: [PATCH 2/3] ci: same ceph version in toolbox and cluster-test Let's use same ceph version(latest Reef) in both cluster-test and toolbox.yaml so that we don't need to pull image twice. Alos, github action helper script was calling `deploy_manifest_with_local_build` which is required for operator.yaml and not for toolbox.yaml. Signed-off-by: subhamkrai (cherry picked from commit b9c41fd548bc6b14cb89405efdaba3334b9a039f) --- tests/scripts/github-action-helper.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 311bee7f3..c40d9e2e0 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -236,6 +236,12 @@ function deploy_manifest_with_local_build() { kubectl create -f $1 } +# Deploy toolbox with same ceph version as the cluster-test for ci +function deploy_toolbox() { + sed -i 's/image: quay\.io\/ceph\/ceph:.*/image: quay.io\/ceph\/ceph:v18/' toolbox.yaml + kubectl create -f toolbox.yaml +} + function replace_ceph_image() { local file="$1" # parameter 1: the file in which to replace the ceph image local ceph_image="${2:-}" # parameter 2: the new ceph image to use @@ -282,7 +288,7 @@ function deploy_cluster() { kubectl create -f filesystem-mirror.yaml kubectl create -f nfs-test.yaml kubectl create -f subvolumegroup.yaml - deploy_manifest_with_local_build toolbox.yaml + deploy_toolbox } function deploy_csi_hostnetwork_disabled_cluster() { @@ -300,7 +306,7 @@ function deploy_csi_hostnetwork_disabled_cluster() { kubectl create -f nfs-test.yaml kubectl create -f cluster-test.yaml kubectl create -f filesystem-test.yaml - deploy_manifest_with_local_build toolbox.yaml + deploy_toolbox } function wait_for_prepare_pod() { @@ -399,7 +405,7 @@ function deploy_first_rook_cluster() { yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"1 kubectl create -f cluster-test.yaml - deploy_manifest_with_local_build toolbox.yaml + deploy_toolbox } function deploy_second_rook_cluster() { @@ -411,7 +417,7 @@ function deploy_second_rook_cluster() { yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external" kubectl create -f cluster-test.yaml yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary - deploy_manifest_with_local_build toolbox.yaml + deploy_toolbox } function wait_for_rgw() { @@ -590,7 +596,7 @@ function deploy_multus_cluster() { cd deploy/examples sed -i 's/.*ROOK_CSI_ENABLE_NFS:.*/ ROOK_CSI_ENABLE_NFS: \"true\"/g' operator.yaml deploy_manifest_with_local_build operator.yaml - deploy_manifest_with_local_build toolbox.yaml + deploy_toolbox sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-multus-test.yaml kubectl create -f cluster-multus-test.yaml kubectl create -f filesystem-test.yaml From 1661f841795f63cf2e0094436c4f869e47ea3791 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Wed, 2 Aug 2023 12:40:26 +0530 Subject: [PATCH 3/3] multus: use right interface in ci validation runner version `2.306` had the interface `net` but somehow version `2.307.1` which is latest doesn't have `net` it has `eth0*` so using that. ``` cat /proc/net/dev Inter-| Receive | Transmit face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed lo: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 tunl0: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 eth0: 446 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 sh-4.4# grep etho /proc/net/dev sh-4.4# grep eth0 /proc/net/dev eth0: 446 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 sh-4.4# exit ``` Signed-off-by: subhamkrai (cherry picked from commit 4502ea8ee141834a87b91fed3f41517f1d9285f6) --- tests/scripts/validate_cluster.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/validate_cluster.sh b/tests/scripts/validate_cluster.sh index fa2b3a749..c918ed119 100755 --- a/tests/scripts/validate_cluster.sh +++ b/tests/scripts/validate_cluster.sh @@ -111,9 +111,9 @@ function test_csi { fi if [ -n "$IS_MULTUS" ]; then echo "verifying csi holder interfaces (multus ones must be present)" - kubectl -n rook-ceph exec -t ds/csi-rbdplugin-holder-my-cluster -- grep net /proc/net/dev - kubectl -n rook-ceph exec -t ds/csi-cephfsplugin-holder-my-cluster -- grep net /proc/net/dev - kubectl -n rook-ceph exec -t ds/csi-nfsplugin-holder-my-cluster -- grep net /proc/net/dev + kubectl -n rook-ceph exec -t ds/csi-rbdplugin-holder-my-cluster -- grep eth0 /proc/net/dev + kubectl -n rook-ceph exec -t ds/csi-cephfsplugin-holder-my-cluster -- grep eth0 /proc/net/dev + kubectl -n rook-ceph exec -t ds/csi-nfsplugin-holder-my-cluster -- grep eth0 /proc/net/dev fi EOF }