diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 0c02870f8..5c0b0899b 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -70,6 +70,21 @@ jobs: mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr) timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done" + - name: test osd.0 auth recovery from keyring file + run: | + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + osd_id=0 + osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + if [ $osd_pod ]; then + timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth del osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be deleted'; done"; + kubectl -n rook-ceph delete pod $osd_pod; + timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth get osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be recovered'; done"; + osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}'); + kubectl -n rook-ceph wait --for=condition=Ready pod/$osd_pod --timeout=120s; + else + echo "osd $osd_id not found, skipping test"; + fi + - name: test external script create-external-cluster-resources.py run: | toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 350348bb2..a1dbe8366 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -109,6 +109,7 @@ CEPH_FSID=%s OSD_UUID=%s OSD_STORE_FLAG="%s" OSD_DATA_DIR=/var/lib/ceph/osd/ceph-"$OSD_ID" +KEYRING_FILE="$OSD_DATA_DIR"/keyring CV_MODE=%s DEVICE="$%s" @@ -135,8 +136,38 @@ with open('/etc/ceph/ceph.conf', 'w') as configfile: config.write(configfile) " -# create new keyring -ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring +# In rare cases keyring file created with prepare-osd but did not +# being stored in ceph auth system therefore we need to import it +# from keyring file instead of creating new one +if ! ceph -n client.admin auth get osd."$OSD_ID" -k /etc/ceph/admin-keyring-store/keyring; then + if [ -f "$KEYRING_FILE" ]; then + # import keyring from existing file + TMP_DIR=$(mktemp -d) + + python3 -c " +import configparser + +config = configparser.ConfigParser() +config.read('$KEYRING_FILE') + +if not config.has_section('osd.$OSD_ID'): + exit() + +config['osd.$OSD_ID'] = {'key': config['osd.$OSD_ID']['key'], 'caps mon': '\"allow profile osd\"', 'caps mgr': '\"allow profile osd\"', 'caps osd': '\"allow *\"'} + +with open('$TMP_DIR/keyring', 'w') as configfile: + config.write(configfile) +" + + cat "$TMP_DIR"/keyring + ceph -n client.admin auth import -i "$TMP_DIR"/keyring -k /etc/ceph/admin-keyring-store/keyring + + rm --recursive --force "$TMP_DIR" + else + # create new keyring if no keyring file found + ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring + fi +fi # active the osd with ceph-volume if [[ "$CV_MODE" == "lvm" ]]; then