Merge pull request #7891 from leseb/wait-for-prom

ceph: wait for the prometheus module to be ready

.github/workflows/canary-integration-test.yml

@@ -54,14 +54,17 @@ jobs:
 
     - name: wait for ceph to be ready
       run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready all 2
-        
+
     - name: test external script create-external-cluster-resources.py
       run: |
-        kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- /bin/bash -c "echo \"$(kubectl get pods -o wide -n rook-ceph -l app=rook-ceph-mgr --no-headers=true|head -n1|awk '{print $6"\t"$1}')\" >>/etc/hosts"
-        kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- mkdir -p /etc/ceph/test-data
-        kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/test-data/ceph-status-out $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}'):/etc/ceph/test-data/
-        kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/create-external-cluster-resources.py $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}'):/etc/ceph
-        kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool
+        toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}')
+        mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr)
+        timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done"
+        kubectl -n rook-ceph exec $toolbox -- /bin/bash -c "echo \"$(kubectl get pods -o wide -n rook-ceph -l app=rook-ceph-mgr --no-headers=true|head -n1|awk '{print $6"\t"$1}')\" >>/etc/hosts"
+        kubectl -n rook-ceph exec $toolbox -- mkdir -p /etc/ceph/test-data
+        kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/test-data/ceph-status-out $toolbox:/etc/ceph/test-data/
+        kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/create-external-cluster-resources.py $toolbox:/etc/ceph
+        kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool
 
     - name: run external script create-external-cluster-resources.py unit tests
       run: |
@@ -122,7 +125,7 @@ jobs:
       run: |
         BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
         tests/scripts/localPathPV.sh "$BLOCK"
-        tests/scripts/github-action-helper.sh create_cluster_prerequisites 
+        tests/scripts/github-action-helper.sh create_cluster_prerequisites
 
     - name: deploy cluster
       run: |
@@ -189,7 +192,7 @@ jobs:
 
     - name: create bluestore partitions and PVCs
       run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs
-        
+
 
     - name: print k8s cluster status
       run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
@@ -198,7 +201,7 @@ jobs:
       run: tests/scripts/github-action-helper.sh build_rook
 
     - name: create cluster prerequisites
-      run: tests/scripts/github-action-helper.sh create_cluster_prerequisites 
+      run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
 
     - name: deploy cluster
       run: |
@@ -210,10 +213,10 @@ jobs:
 
     - name: wait for prepare pod
       run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
-        
+
     - name: wait for ceph to be ready
       run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
-        
+
     - name: Upload pvc-db test result
       uses: actions/upload-artifact@v2
       if: always()
@@ -252,7 +255,7 @@ jobs:
 
     - name: use local disk
       run: tests/scripts/github-action-helper.sh use_local_disk
-        
+
     - name: create bluestore partitions and PVCs for wal
       run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal
 
@@ -330,7 +333,7 @@ jobs:
     - name: create cluster prerequisites
       run: |
         tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1)
-        tests/scripts/github-action-helper.sh create_cluster_prerequisites 
+        tests/scripts/github-action-helper.sh create_cluster_prerequisites
 
     - name: deploy cluster
       run: |
@@ -398,7 +401,7 @@ jobs:
       run: tests/scripts/github-action-helper.sh build_rook
 
     - name: create cluster prerequisites
-      run: tests/scripts/github-action-helper.sh create_cluster_prerequisites 
+      run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
 
     - name: deploy cluster
       run: |
@@ -457,7 +460,7 @@ jobs:
 
     - name: create bluestore partitions and PVCs for wal
       run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal
-        
+
     - name: print k8s cluster status
       run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
 
@@ -533,7 +536,7 @@ jobs:
     - name: create cluster prerequisites
       run: |
         tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1)
-        tests/scripts/github-action-helper.sh create_cluster_prerequisites 
+        tests/scripts/github-action-helper.sh create_cluster_prerequisites
 
     - name: deploy vault
       run: tests/scripts/deploy-validate-vault.sh deploy
@@ -616,7 +619,7 @@ jobs:
 
     - name: create LV on disk
       run: tests/scripts/github-action-helper.sh create_LV_on_disk
-        
+
     - name: deploy cluster
       run: |
         kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml