From b035c5d18bf88965b4958ce0e70c31be26b4449a Mon Sep 17 00:00:00 2001
From: subhamkrai <srai@redhat.com>
Date: Tue, 25 Jul 2023 16:48:07 +0530
Subject: [PATCH 1/4] ci: add detached mode in tmate

with detached mode, we don't need to wait for the
tests to complete. Now, we can get inside tmate before
the tests start and we can monitor the tests.

By default, this mode will wait at the end of the job for
a user to connect and then to terminate the tmate session.
If no user has connected within 10 minutes after the post-job step started,
it will terminate the tmate session and quit gracefully.

Signed-off-by: subhamkrai <srai@redhat.com>
(cherry picked from commit f171e1c037626c58590fba1aae1d06ff2385fd8c)
---
 .github/workflows/build.yml                   |  37 +-
 .github/workflows/canary-integration-test.yml | 333 ++++++------------
 .../integration-test-helm-suite.yaml          |  18 +-
 .../workflows/integration-test-mgr-suite.yaml |  18 +-
 .../integration-test-multi-cluster-suite.yaml |  18 +-
 .../integration-test-object-suite.yaml        |  18 +-
 .../integration-test-smoke-suite.yaml         |  18 +-
 .../integration-test-upgrade-suite.yaml       |  28 +-
 .github/workflows/push-build.yaml             |  20 +-
 .github/workflows/tmate_debug/action.yml      |  23 ++
 10 files changed, 178 insertions(+), 353 deletions(-)
 create mode 100644 .github/workflows/tmate_debug/action.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5905eed40..09a33e49f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -25,7 +25,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-go@v3
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
+      - uses: actions/setup-go@v4
         with:
           go-version: 1.19
 
@@ -71,18 +76,6 @@ jobs:
       - name: validate gen-rbac
         run: tests/scripts/validate_modified_files.sh gen-rbac
 
-      - name: consider debugging
-        if: failure()
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   linux-build-all:
     runs-on: ubuntu-20.04
@@ -97,6 +90,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup golang ${{ matrix.go-version }}
         uses: actions/setup-go@v3
         with:
@@ -110,16 +108,3 @@ jobs:
       - name: build.all rook with go ${{ matrix.go-version }}
         run: |
           tests/scripts/github-action-helper.sh build_rook_all
-
-      - name: consider debugging
-        if: failure()
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index fcb014a4e..129b6bca4 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -31,6 +31,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -219,18 +224,6 @@ jobs:
         with:
           name: canary
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   raw-disk:
     runs-on: ubuntu-20.04
@@ -241,6 +234,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -279,18 +277,6 @@ jobs:
         with:
           name: canary
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   two-osds-in-device:
     runs-on: ubuntu-20.04
@@ -300,6 +286,10 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
 
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
@@ -333,18 +323,6 @@ jobs:
         with:
           name: canary
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   osd-with-metadata-device:
     runs-on: ubuntu-20.04
@@ -355,6 +333,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -393,18 +376,6 @@ jobs:
         with:
           name: canary
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   encryption:
     runs-on: ubuntu-20.04
@@ -415,6 +386,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -447,11 +423,6 @@ jobs:
         with:
           name: canary
 
-      - name: setup tmate session for debugging when event is PR
-        if: failure() && github.event_name == 'pull_request'
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
-
   lvm:
     runs-on: ubuntu-20.04
     if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
@@ -461,6 +432,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -498,18 +474,6 @@ jobs:
         with:
           name: canary
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   pvc:
     runs-on: ubuntu-20.04
@@ -520,6 +484,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -578,18 +547,6 @@ jobs:
         with:
           name: pvc
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   pvc-db:
     runs-on: ubuntu-20.04
@@ -600,6 +557,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -634,18 +596,6 @@ jobs:
         with:
           name: pvc-db
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   pvc-db-wal:
     runs-on: ubuntu-20.04
@@ -656,6 +606,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -693,18 +648,6 @@ jobs:
         with:
           name: pvc-db-wal
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   encryption-pvc:
     runs-on: ubuntu-20.04
@@ -715,6 +658,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -764,18 +712,6 @@ jobs:
         with:
           name: encryption-pvc
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   encryption-pvc-db:
     runs-on: ubuntu-20.04
@@ -786,6 +722,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -822,18 +763,6 @@ jobs:
         with:
           name: encryption-pvc-db
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   encryption-pvc-db-wal:
     runs-on: ubuntu-20.04
@@ -844,6 +773,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -890,18 +824,6 @@ jobs:
         with:
           name: encryption-pvc-db-wal
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   encryption-pvc-kms-vault-token-auth:
     runs-on: ubuntu-20.04
@@ -912,6 +834,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -979,18 +906,6 @@ jobs:
         with:
           name: encryption-pvc-kms-vault-token-auth
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   encryption-pvc-kms-vault-k8s-auth:
     runs-on: ubuntu-20.04
@@ -1001,6 +916,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -1048,18 +968,6 @@ jobs:
         with:
           name: encryption-pvc-kms-vault-k8s-auth
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   lvm-pvc:
     runs-on: ubuntu-20.04
@@ -1070,6 +978,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -1109,18 +1022,6 @@ jobs:
         with:
           name: lvm-pvc
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   multi-cluster-mirroring:
     runs-on: ubuntu-20.04
@@ -1131,6 +1032,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -1371,18 +1277,7 @@ jobs:
         with:
           name: multi-cluster-mirroring
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
 
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   rgw-multisite-testing:
     runs-on: ubuntu-20.04
@@ -1393,6 +1288,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: run RGW multisite test
         uses: ./.github/workflows/rgw-multisite-test
         with:
@@ -1406,18 +1307,7 @@ jobs:
           name: rgw-multisite-testing
           path: test
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
 
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   encryption-pvc-kms-ibm-kp:
     runs-on: ubuntu-20.04
@@ -1426,6 +1316,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: run encryption KMS IBM Key Protect
         uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
         if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
@@ -1444,18 +1340,7 @@ jobs:
           name: encryption-pvc-kms-ibm-kp
           path: test
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
 
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   multus-cluster-network:
     runs-on: ubuntu-20.04
@@ -1466,6 +1351,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup golang
         uses: actions/setup-go@v3
         with:
@@ -1529,18 +1420,7 @@ jobs:
         with:
           name: canary-multus
 
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
 
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
 
   csi-hostnetwork-disabled:
     runs-on: ubuntu-20.04
@@ -1551,6 +1431,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -1587,16 +1473,3 @@ jobs:
         uses: ./.github/workflows/collect-logs
         with:
           name: csi-hostnetwork-disabled
-
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml
index 4e79a8a6b..4d4e45d17 100644
--- a/.github/workflows/integration-test-helm-suite.yaml
+++ b/.github/workflows/integration-test-helm-suite.yaml
@@ -29,6 +29,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup latest cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
@@ -60,16 +65,3 @@ jobs:
         with:
           name: ceph-helm-suite-artifact-${{ matrix.kubernetes-versions }}
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
-
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml
index eb9170c58..5ad887ecb 100644
--- a/.github/workflows/integration-test-mgr-suite.yaml
+++ b/.github/workflows/integration-test-mgr-suite.yaml
@@ -28,6 +28,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
@@ -54,16 +59,3 @@ jobs:
         with:
           name: ceph-mgr-suite-artifact-${{ matrix.kubernetes-versions }}
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
-
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml
index 13838c0a8..3eb982822 100644
--- a/.github/workflows/integration-test-multi-cluster-suite.yaml
+++ b/.github/workflows/integration-test-multi-cluster-suite.yaml
@@ -29,6 +29,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
@@ -56,16 +61,3 @@ jobs:
         with:
           name: ceph-multi-cluster-deploy-suite-artifact-${{ matrix.kubernetes-versions }}
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
-
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml
index 74df6da03..33fda393a 100644
--- a/.github/workflows/integration-test-object-suite.yaml
+++ b/.github/workflows/integration-test-object-suite.yaml
@@ -29,6 +29,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup latest cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
@@ -55,16 +60,3 @@ jobs:
         with:
           name: ceph-object-suite-artifact-${{ matrix.kubernetes-versions }}
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
-
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml
index f09f241c1..ac6dfe2d0 100644
--- a/.github/workflows/integration-test-smoke-suite.yaml
+++ b/.github/workflows/integration-test-smoke-suite.yaml
@@ -29,6 +29,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
@@ -55,16 +60,3 @@ jobs:
         with:
           name: ceph-smoke-suite-artifact-${{ matrix.kubernetes-versions }}
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
-
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml
index f93ec30ab..4705a0053 100644
--- a/.github/workflows/integration-test-upgrade-suite.yaml
+++ b/.github/workflows/integration-test-upgrade-suite.yaml
@@ -29,6 +29,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
@@ -56,11 +61,6 @@ jobs:
           name: ceph-upgrade-suite-artifact-${{ matrix.kubernetes-versions }}
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
 
-      - name: setup tmate session for debugging when event is PR
-        if: failure() && github.event_name == 'pull_request'
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
-
   TestHelmUpgradeSuite:
     if: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && !contains(github.event.pull_request.labels.*.name, 'skip-ci') }}
     runs-on: ubuntu-20.04
@@ -74,6 +74,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
@@ -103,16 +108,3 @@ jobs:
         with:
           name: ceph-upgrade-helm-suite-artifact-${{ matrix.kubernetes-versions }}
           path: /home/runner/work/rook/rook/tests/integration/_output/tests/
-
-      - name: consider debugging
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/push-build.yaml b/.github/workflows/push-build.yaml
index cddb8bdb6..af5022713 100644
--- a/.github/workflows/push-build.yaml
+++ b/.github/workflows/push-build.yaml
@@ -25,7 +25,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-go@v3
+      - name: consider debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+
+      - uses: actions/setup-go@v4
         with:
           go-version: 1.19
 
@@ -75,16 +80,3 @@ jobs:
           GITHUB_REF: $ {{ env.GITHUB_REF }}
         run: |
           tests/scripts/build-release.sh
-
-      - name: consider debugging
-        if: failure()
-        run: |
-          # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
-          if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ secrets.USE_TMATE }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
-            echo USE_TMATE=1 >> $GITHUB_ENV
-          fi
-
-      - name: set up tmate session for debugging
-        if: failure() && env.USE_TMATE
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
diff --git a/.github/workflows/tmate_debug/action.yml b/.github/workflows/tmate_debug/action.yml
new file mode 100644
index 000000000..b547188da
--- /dev/null
+++ b/.github/workflows/tmate_debug/action.yml
@@ -0,0 +1,23 @@
+name: "Tmate debugging tests"
+description: "Setup tmate session if the test fails"
+inputs:
+  use-tmate:
+    description: "boolean for enabling TMATE"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: consider debugging
+      shell: bash --noprofile --norc -eo pipefail -x {0}
+      run: |
+        # Enable tmate only in the Rook fork, where the USE_TMATE secret is set in the repo, or if the action is re-run
+        if [ "$GITHUB_REPOSITORY_OWNER" = "rook" ] || [ -n "${{ inputs.use-tmate }}" ] || [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then
+          echo USE_TMATE=1 >> $GITHUB_ENV
+        fi
+
+    - name: set up tmate session for debugging
+      if: env.USE_TMATE
+      uses: mxschmitt/action-tmate@v3
+      with:
+        limit-access-to-actor: false
+        detached: true

From 343c26d5e9494b43a40d073a12804b4412529f73 Mon Sep 17 00:00:00 2001
From: subhamkrai <srai@redhat.com>
Date: Tue, 1 Aug 2023 17:46:14 +0530
Subject: [PATCH 2/4] ci: add github action to free space in runner

adding github action to free space in runner
which will help us avoiding mon low space error

Signed-off-by: subhamkrai <srai@redhat.com>
(cherry picked from commit 334a632ecd34c7bab474b06360a7639aa8f17c2f)
---
 .github/workflows/canary-integration-test.yml | 141 +++++++++++++++++-
 .../integration-test-helm-suite.yaml          |   7 +
 .../workflows/integration-test-mgr-suite.yaml |   7 +
 .../integration-test-multi-cluster-suite.yaml |   7 +
 .../integration-test-object-suite.yaml        |   7 +
 .../integration-test-smoke-suite.yaml         |   7 +
 .../integration-test-upgrade-suite.yaml       |  14 ++
 7 files changed, 189 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index 129b6bca4..abd01b3ae 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -36,6 +36,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -224,7 +231,6 @@ jobs:
         with:
           name: canary
 
-
   raw-disk:
     runs-on: ubuntu-20.04
     if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
@@ -239,6 +245,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -291,6 +304,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -338,6 +358,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -391,6 +418,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -437,6 +471,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -489,6 +530,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -562,6 +610,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -611,6 +666,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -663,6 +725,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -727,6 +796,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -778,6 +854,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -839,6 +922,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -921,6 +1011,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -983,6 +1080,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -1037,6 +1141,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
@@ -1294,6 +1405,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: run RGW multisite test
         uses: ./.github/workflows/rgw-multisite-test
         with:
@@ -1322,6 +1440,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: run encryption KMS IBM Key Protect
         uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
         if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
@@ -1357,6 +1482,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup golang
         uses: actions/setup-go@v3
         with:
@@ -1437,6 +1569,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
         with:
diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml
index 4d4e45d17..b680cb2bf 100644
--- a/.github/workflows/integration-test-helm-suite.yaml
+++ b/.github/workflows/integration-test-helm-suite.yaml
@@ -34,6 +34,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup latest cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml
index 5ad887ecb..ec9368174 100644
--- a/.github/workflows/integration-test-mgr-suite.yaml
+++ b/.github/workflows/integration-test-mgr-suite.yaml
@@ -33,6 +33,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml
index 3eb982822..27df1a3b0 100644
--- a/.github/workflows/integration-test-multi-cluster-suite.yaml
+++ b/.github/workflows/integration-test-multi-cluster-suite.yaml
@@ -34,6 +34,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml
index 33fda393a..abf9e8d1c 100644
--- a/.github/workflows/integration-test-object-suite.yaml
+++ b/.github/workflows/integration-test-object-suite.yaml
@@ -34,6 +34,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup latest cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml
index ac6dfe2d0..e2dedc8a5 100644
--- a/.github/workflows/integration-test-smoke-suite.yaml
+++ b/.github/workflows/integration-test-smoke-suite.yaml
@@ -34,6 +34,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml
index 4705a0053..cffc6a8ce 100644
--- a/.github/workflows/integration-test-upgrade-suite.yaml
+++ b/.github/workflows/integration-test-upgrade-suite.yaml
@@ -34,6 +34,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:
@@ -79,6 +86,13 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:

From 3e9f8517f6a715c4725fe291c86d8a6ab0750fdc Mon Sep 17 00:00:00 2001
From: subhamkrai <srai@redhat.com>
Date: Wed, 2 Aug 2023 12:35:29 +0530
Subject: [PATCH 3/4] ci: same ceph version in toolbox and cluster-test

Let's use same ceph version(latest Reef) in both
cluster-test and toolbox.yaml so that we don't need
to pull image twice. Alos, github action helper script
was calling `deploy_manifest_with_local_build` which
is required for operator.yaml and not for toolbox.yaml.

Signed-off-by: subhamkrai <srai@redhat.com>
(cherry picked from commit b9c41fd548bc6b14cb89405efdaba3334b9a039f)
---
 tests/scripts/github-action-helper.sh | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh
index d40206d19..df18e23c1 100755
--- a/tests/scripts/github-action-helper.sh
+++ b/tests/scripts/github-action-helper.sh
@@ -222,6 +222,12 @@ function deploy_manifest_with_local_build() {
   kubectl create -f $1
 }
 
+# Deploy toolbox with same ceph version as the cluster-test for ci
+function deploy_toolbox() {
+  sed -i 's/image: quay\.io\/ceph\/ceph:.*/image: quay.io\/ceph\/ceph:v18/' toolbox.yaml
+  kubectl create -f toolbox.yaml
+}
+
 function replace_ceph_image() {
   local file="$1"           # parameter 1: the file in which to replace the ceph image
   local ceph_image="${2:-}" # parameter 2: the new ceph image to use
@@ -268,7 +274,7 @@ function deploy_cluster() {
   kubectl create -f filesystem-mirror.yaml
   kubectl create -f nfs-test.yaml
   kubectl create -f subvolumegroup.yaml
-  deploy_manifest_with_local_build toolbox.yaml
+  deploy_toolbox
 }
 
 function deploy_csi_hostnetwork_disabled_cluster() {
@@ -286,7 +292,7 @@ function deploy_csi_hostnetwork_disabled_cluster() {
   kubectl create -f nfs-test.yaml
   kubectl create -f cluster-test.yaml
   kubectl create -f filesystem-test.yaml
-  deploy_manifest_with_local_build toolbox.yaml
+  deploy_toolbox
 }
 
 function wait_for_prepare_pod() {
@@ -386,7 +392,7 @@ function deploy_first_rook_cluster() {
   yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false
   yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"1
   kubectl create -f cluster-test.yaml
-  deploy_manifest_with_local_build toolbox.yaml
+  deploy_toolbox
 }
 
 function deploy_second_rook_cluster() {
@@ -398,7 +404,7 @@ function deploy_second_rook_cluster() {
   yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external"
   kubectl create -f cluster-test.yaml
   yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary
-  deploy_manifest_with_local_build toolbox.yaml
+  deploy_toolbox
 }
 
 function wait_for_rgw() {
@@ -577,7 +583,7 @@ function deploy_multus_cluster() {
   cd deploy/examples
   sed -i 's/.*ROOK_CSI_ENABLE_NFS:.*/  ROOK_CSI_ENABLE_NFS: \"true\"/g' operator.yaml
   deploy_manifest_with_local_build operator.yaml
-  deploy_manifest_with_local_build toolbox.yaml
+  deploy_toolbox
   sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-multus-test.yaml
   kubectl create -f cluster-multus-test.yaml
   kubectl create -f filesystem-test.yaml

From b8880be365fab972568ecd1561bbc6c051876f0d Mon Sep 17 00:00:00 2001
From: subhamkrai <srai@redhat.com>
Date: Wed, 2 Aug 2023 12:40:26 +0530
Subject: [PATCH 4/4] multus: use right interface in ci validation

runner version `2.306` had the interface `net`
but somehow version `2.307.1` which is latest
doesn't have `net` it has `eth0*` so using that.

```
cat /proc/net/dev
Inter-|   Receive                                                |  Transmit
 face |bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed
    lo:       0       0    0    0    0     0          0         0        0       0    0    0    0     0       0          0
 tunl0:       0       0    0    0    0     0          0         0        0       0    0    0    0     0       0          0
  eth0:     446       5    0    0    0     0          0         0        0       0    0    0    0     0       0          0
sh-4.4# grep etho /proc/net/dev
sh-4.4# grep eth0 /proc/net/dev
  eth0:     446       5    0    0    0     0          0         0        0       0    0    0    0     0       0          0
sh-4.4# exit
```

Signed-off-by: subhamkrai <srai@redhat.com>
(cherry picked from commit 4502ea8ee141834a87b91fed3f41517f1d9285f6)
---
 tests/scripts/validate_cluster.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/scripts/validate_cluster.sh b/tests/scripts/validate_cluster.sh
index fa2b3a749..c918ed119 100755
--- a/tests/scripts/validate_cluster.sh
+++ b/tests/scripts/validate_cluster.sh
@@ -111,9 +111,9 @@ function test_csi {
     fi
     if [ -n "$IS_MULTUS" ]; then
       echo "verifying csi holder interfaces (multus ones must be present)"
-      kubectl -n rook-ceph exec -t ds/csi-rbdplugin-holder-my-cluster -- grep net /proc/net/dev
-      kubectl -n rook-ceph exec -t ds/csi-cephfsplugin-holder-my-cluster -- grep net /proc/net/dev
-      kubectl -n rook-ceph exec -t ds/csi-nfsplugin-holder-my-cluster -- grep net /proc/net/dev
+      kubectl -n rook-ceph exec -t ds/csi-rbdplugin-holder-my-cluster -- grep eth0 /proc/net/dev
+      kubectl -n rook-ceph exec -t ds/csi-cephfsplugin-holder-my-cluster -- grep eth0 /proc/net/dev
+      kubectl -n rook-ceph exec -t ds/csi-nfsplugin-holder-my-cluster -- grep eth0 /proc/net/dev
     fi
 EOF
 }