From cc763df671170882f3b426e69115e58c11a105e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 12 May 2021 14:57:47 +0200 Subject: [PATCH] ceph: wait for the prometheus module to be ready MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before running the external script test let's make sure the prometheus exporter is loaded. We often see in the CI errors like: ``` kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool shell: /bin/bash -e {0} env: GOROOT: /opt/hostedtoolcache/go/1.16.4/x64 MINIKUBE_HOME: /home/runner/work/_temp Execution Failed: unable to connect to endpoint: 172.17.0.10:9283 Traceback (most recent call last): File "/etc/ceph/create-external-cluster-resources.py", line 801, in raise err File "/etc/ceph/create-external-cluster-resources.py", line 798, in rjObj.main() File "/etc/ceph/create-external-cluster-resources.py", line 779, in main generated_output = self.gen_json_out() File "/etc/ceph/create-external-cluster-resources.py", line 613, in gen_json_out self._gen_output_map() File "/etc/ceph/create-external-cluster-resources.py", line 598, in _gen_output_map self.out_map['MONITORING_ENDPOINT_PORT'] = self.get_active_and_standby_mgrs() File "/etc/ceph/create-external-cluster-resources.py", line 389, in get_active_and_standby_mgrs self.endpoint_dial(monitoring_endpoint) File "/etc/ceph/create-external-cluster-resources.py", line 245, in endpoint_dial "unable to connect to endpoint: {}".format(endpoint_str)) __main__.ExecutionFailureException: unable to connect to endpoint: 172.17.0.10:9283 command terminated with exit code 1 ``` but then when we log into the runner, we can see the command works. So let's wait a little bit for the module to be up. Signed-off-by: Sébastien Han --- .github/workflows/canary-integration-test.yml | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 996bb31c8..dfab27864 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -54,14 +54,17 @@ jobs: - name: wait for ceph to be ready run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready all 2 - + - name: test external script create-external-cluster-resources.py run: | - kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- /bin/bash -c "echo \"$(kubectl get pods -o wide -n rook-ceph -l app=rook-ceph-mgr --no-headers=true|head -n1|awk '{print $6"\t"$1}')\" >>/etc/hosts" - kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- mkdir -p /etc/ceph/test-data - kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/test-data/ceph-status-out $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}'):/etc/ceph/test-data/ - kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/create-external-cluster-resources.py $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}'):/etc/ceph - kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') + mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr) + timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done" + kubectl -n rook-ceph exec $toolbox -- /bin/bash -c "echo \"$(kubectl get pods -o wide -n rook-ceph -l app=rook-ceph-mgr --no-headers=true|head -n1|awk '{print $6"\t"$1}')\" >>/etc/hosts" + kubectl -n rook-ceph exec $toolbox -- mkdir -p /etc/ceph/test-data + kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/test-data/ceph-status-out $toolbox:/etc/ceph/test-data/ + kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/create-external-cluster-resources.py $toolbox:/etc/ceph + kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool - name: run external script create-external-cluster-resources.py unit tests run: | @@ -122,7 +125,7 @@ jobs: run: | BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) tests/scripts/localPathPV.sh "$BLOCK" - tests/scripts/github-action-helper.sh create_cluster_prerequisites + tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy cluster run: | @@ -189,7 +192,7 @@ jobs: - name: create bluestore partitions and PVCs run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs - + - name: print k8s cluster status run: tests/scripts/github-action-helper.sh print_k8s_cluster_status @@ -198,7 +201,7 @@ jobs: run: tests/scripts/github-action-helper.sh build_rook - name: create cluster prerequisites - run: tests/scripts/github-action-helper.sh create_cluster_prerequisites + run: tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy cluster run: | @@ -210,10 +213,10 @@ jobs: - name: wait for prepare pod run: tests/scripts/github-action-helper.sh wait_for_prepare_pod - + - name: wait for ceph to be ready run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - + - name: Upload pvc-db test result uses: actions/upload-artifact@v2 if: always() @@ -252,7 +255,7 @@ jobs: - name: use local disk run: tests/scripts/github-action-helper.sh use_local_disk - + - name: create bluestore partitions and PVCs for wal run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal @@ -330,7 +333,7 @@ jobs: - name: create cluster prerequisites run: | tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1) - tests/scripts/github-action-helper.sh create_cluster_prerequisites + tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy cluster run: | @@ -398,7 +401,7 @@ jobs: run: tests/scripts/github-action-helper.sh build_rook - name: create cluster prerequisites - run: tests/scripts/github-action-helper.sh create_cluster_prerequisites + run: tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy cluster run: | @@ -457,7 +460,7 @@ jobs: - name: create bluestore partitions and PVCs for wal run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal - + - name: print k8s cluster status run: tests/scripts/github-action-helper.sh print_k8s_cluster_status @@ -533,7 +536,7 @@ jobs: - name: create cluster prerequisites run: | tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1) - tests/scripts/github-action-helper.sh create_cluster_prerequisites + tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy vault run: tests/scripts/deploy-validate-vault.sh deploy @@ -616,7 +619,7 @@ jobs: - name: create LV on disk run: tests/scripts/github-action-helper.sh create_LV_on_disk - + - name: deploy cluster run: | kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml