osd: add osd auth recovery from file canary job test

Add test to canary-integration-test scenarios which removes
osd auth and then recovers it on osd activation from keyring
file.

Resolves: #14825
Signed-off-by: Peter Razumovsky <prazumovsky@mirantis.com>
(cherry picked from commit 12613bc2fa)

.github/workflows/canary-integration-test.yml

@@ -70,6 +70,21 @@ jobs:
           mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr)
           timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done"
 
+      - name: test osd.0 auth recovery from keyring file
+        run: |
+          toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
+          osd_id=0
+          osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
+          if [ $osd_pod ]; then
+            timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth del osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be deleted'; done";
+            kubectl -n rook-ceph delete pod $osd_pod;
+            timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth get osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be recovered'; done";
+            osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}');
+            kubectl -n rook-ceph wait --for=condition=Ready pod/$osd_pod --timeout=120s;
+          else
+            echo "osd $osd_id not found, skipping test";
+          fi
+
       - name: test external script create-external-cluster-resources.py
         run: |
           toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')

pkg/operator/ceph/cluster/osd/spec.go

@@ -113,7 +113,6 @@ KEYRING_FILE="$OSD_DATA_DIR"/keyring
 CV_MODE=%s
 DEVICE="$%s"
 
-<<<<<<< HEAD
 # "ceph.conf" must have the "fsid" global configuration to activate encrypted OSDs
 # after the following Ceph's PR is merged.
 # https://github.com/ceph/ceph/commit/25655e5a8829e001adf467511a6bde8142b0a575
@@ -137,9 +136,6 @@ with open('/etc/ceph/ceph.conf', 'w') as configfile:
     config.write(configfile)
 "
 
-# create new keyring
-ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring
-=======
 # In rare cases keyring file created with prepare-osd but did not
 # being stored in ceph auth system therefore we need to import it
 # from keyring file instead of creating new one
@@ -172,7 +168,6 @@ with open('$TMP_DIR/keyring', 'w') as configfile:
         ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring
     fi
 fi
->>>>>>> 6eb2d1121 (osd: import keyring file on activate to ceph auth if not imported yet)
 
 # active the osd with ceph-volume
 if [[ "$CV_MODE" == "lvm" ]]; then