ci: add test for key rotation

This commit adds test for key rotation.

Signed-off-by: Rakshith R <rar@redhat.com>
(cherry picked from commit b9fc9a643e)

.github/workflows/canary-integration-test.yml

@@ -744,11 +744,6 @@ jobs:
           kubectl -n rook-ceph get secrets
           sudo lsblk
 
-      - name: test osd deployment removal and re-hydration
-        run: |
-          kubectl -n rook-ceph delete deploy/rook-ceph-osd-0
-          tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
-
       - name: teardown cluster with cleanup policy
         run: |
           kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}'
@@ -866,6 +861,7 @@ jobs:
           cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
           cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
+          kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}'
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
 
       - name: wait for prepare pod
@@ -877,6 +873,14 @@ jobs:
           kubectl -n rook-ceph get pods
           kubectl -n rook-ceph get secrets
 
+      - name: wait and verify key rotation
+        run: tests/scripts/github-action-helper.sh verify_key_rotation
+
+      - name: test osd deployment removal and re-hydration
+        run: |
+          kubectl -n rook-ceph delete deploy/rook-ceph-osd-0
+          tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
+
       - name: collect common logs
         if: always()
         uses: ./.github/workflows/collect-logs

tests/scripts/github-action-helper.sh

@@ -329,6 +329,31 @@ function wait_for_ceph_to_be_ready() {
   kubectl -n rook-ceph get pods
 }
 
+function verify_key_rotation(){
+  pvc_name=$(kubectl get pvc -n rook-ceph -l ceph.rook.io/setIndex=0 -o jsonpath='{.items[0].metadata.name}')
+  old_key=$(kubectl -n rook-ceph get secrets -l "pvc_name=$pvc_name" -o jsonpath='{.items[0].data.'dmcrypt-key'}' | base64 --decode)
+  runtime="3 minutes"
+  endtime=$(date -ud "$runtime" +%s)
+  while [[ $(date -u +%s) -le $endtime ]]
+  do
+      echo "Time Now: `date +%H:%M:%S`"
+      new_key=$(kubectl -n rook-ceph get secrets -l "pvc_name=$pvc_name" -o jsonpath='{.items[0].data.'dmcrypt-key'}' | base64 --decode)
+      if [ "$old_key" != "$new_key" ]; then
+        echo "encryption passphrase is successfully rotated"
+        exit 0
+      fi
+      echo "encryption passphrase is not rotated, sleeping for 10 seconds"
+      sleep 10s
+  done
+  new_key=$(kubectl -n rook-ceph get secrets -l "pvc_name=$pvc_name" -o jsonpath='{.items[0].data.'dmcrypt-key'}' | base64 --decode)
+  if [ "$old_key" == "$new_key" ]; then
+    echo "encryption passphrase is not rotated"
+    exit 1
+  else
+    echo "encryption passphrase is successfully rotated"
+  fi
+}
+
 function check_ownerreferences() {
   curl -L https://github.com/kubernetes-sigs/kubectl-check-ownerreferences/releases/download/v0.2.0/kubectl-check-ownerreferences-linux-amd64.tar.gz -o kubectl-check-ownerreferences-linux-amd64.tar.gz
   tar xzvf kubectl-check-ownerreferences-linux-amd64.tar.gz