From b6404ae2ab99ec4181fd29ba57f3fcef8b38f8a8 Mon Sep 17 00:00:00 2001 From: Ali Maredia Date: Tue, 11 May 2021 16:12:10 -0400 Subject: [PATCH] ceph: rgw multisite intgration testing This test: - starts up 2 minikube clusters - create 2 ceph clusters - creates object multisite CRDs on each cluster and syncs the clusters - writes an object to cluster 1 and reads it on cluster 2 This commit also adds new functions in github-action-helper.sh that aid in the multisite test and the multi cluster mirroring test. Signed-off-by: Ali Maredia --- .github/workflows/canary-integration-test.yml | 100 +++++++++++++++--- .../object-multisite-pull-realm-test.yaml | 63 +++++++++++ .../ceph/object-multisite-pull-realm.yaml | 19 ++-- .../ceph/object-multisite-test.yaml | 50 +++++++++ .../kubernetes/ceph/object-multisite.yaml | 7 +- tests/scripts/github-action-helper.sh | 46 ++++++++ 6 files changed, 260 insertions(+), 25 deletions(-) create mode 100644 cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml create mode 100644 cluster/examples/kubernetes/ceph/object-multisite-test.yaml diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 6ac811048..964d12e01 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -720,26 +720,18 @@ jobs: - name: deploy first cluster rook run: | - BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1) + tests/scripts/github-action-helper.sh deploy_first_rook_cluster cd cluster/examples/kubernetes/ceph/ - kubectl create -f crds.yaml -f common.yaml -f operator.yaml - yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false - yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false - yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}1 - kubectl create -f cluster-test.yaml -f rbdmirror.yaml -f filesystem-mirror.yaml -f toolbox.yaml + kubectl create -f rbdmirror.yaml -f filesystem-mirror.yaml # cephfs-mirroring is a push operation # running bootstrap create on secondary and bootstrap import on primary. mirror daemons on primary. - name: deploy second cluster rook run: | - BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1) + tests/scripts/github-action-helper.sh deploy_second_rook_cluster cd cluster/examples/kubernetes/ceph/ - NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f - - sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' cluster-test.yaml rbdmirror.yaml - yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}2 - yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-secondary" - yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary - kubectl create -f cluster-test.yaml -f rbdmirror.yaml -f toolbox.yaml + sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' rbdmirror.yaml + kubectl create -f rbdmirror.yaml - name: wait for ceph cluster 1 to be ready run: | @@ -876,3 +868,85 @@ jobs: if: failure() uses: mxschmitt/action-tmate@v3 timeout-minutes: 60 + + rgw-multisite-testing: + runs-on: ubuntu-18.04 + if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: setup golang + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: install deps + run: | + tests/scripts/github-action-helper.sh install_deps + sudo apt-get install -y s3cmd + + - name: setup minikube + uses: manusa/actions-setup-minikube@v2.3.1 + with: + minikube version: 'v1.18.1' + kubernetes version: 'v1.19.2' + start args: --memory 6g --cpus=2 + github token: ${{ secrets.GITHUB_TOKEN }} + + - name: use local disk into two partitions + run: | + BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/github-action-helper.sh use_local_disk + tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 + sudo lsblk + + - name: build rook + run: tests/scripts/github-action-helper.sh build_rook + + - name: deploy first cluster rook + run: | + tests/scripts/github-action-helper.sh deploy_first_rook_cluster + kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml + # wait for multisite store to be created + tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph + + - name: prep second cluster pull realm config + run: | + cd cluster/examples/kubernetes/ceph/ + IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") + yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80 + BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") + BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") + sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml + sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml + + - name: deploy second cluster rook + run: | + tests/scripts/github-action-helper.sh deploy_second_rook_cluster + kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml + # wait for realms to be pulled and zone-b-multisite-store to be created + tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary + + - name: wait for ceph cluster 1 to be ready + run: | + mkdir test + tests/scripts/validate_cluster.sh osd 1 + kubectl -n rook-ceph get pods + + - name: write an object to one cluster, read from the other + run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2 + + - name: upload test result + uses: actions/upload-artifact@v2 + if: always() + with: + name: rgw-multisite-testing + path: test + + - name: setup tmate session for debugging + if: failure() + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 60 diff --git a/cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml b/cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml new file mode 100644 index 000000000..a11e6072a --- /dev/null +++ b/cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml @@ -0,0 +1,63 @@ +################################################################################################################# +# Create an object store with settings for a test environment. Only a single OSD is required in this example. +# kubectl create -f object-multisite-pull-realm-test.yaml +################################################################################################################# +apiVersion: v1 +kind: Secret +metadata: + name: realm-a-keys + namespace: rook-ceph-secondary +data: + # TODO: Replace with keys for your cluster + # these keys should be the base64 encoded versions of the actual keys or copied from the realm's on the other cluster's secret + access-key: VzFjNFltMVdWRTFJWWxZelZWQT0= + secret-key: WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ== +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectRealm +metadata: + name: realm-a + namespace: rook-ceph-secondary +spec: + # This endpoint in this section needs is an endpoint from the master zone in the master zone group of realm-a. See object-multisite.md for more details. + pull: + endpoint: http://10.103.133.16:80 +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZoneGroup +metadata: + name: zonegroup-a + namespace: rook-ceph-secondary +spec: + realm: realm-a +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZone +metadata: + name: zone-b + namespace: rook-ceph-secondary +spec: + zoneGroup: zonegroup-a + metadataPool: + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false + dataPool: + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: zone-b-multisite-store + namespace: rook-ceph-secondary +spec: + gateway: + port: 80 + # securePort: 443 + instances: 1 + zone: + name: zone-b diff --git a/cluster/examples/kubernetes/ceph/object-multisite-pull-realm.yaml b/cluster/examples/kubernetes/ceph/object-multisite-pull-realm.yaml index 2d678eb9c..ed6cd786e 100644 --- a/cluster/examples/kubernetes/ceph/object-multisite-pull-realm.yaml +++ b/cluster/examples/kubernetes/ceph/object-multisite-pull-realm.yaml @@ -1,12 +1,13 @@ ################################################################################################################# -# Create an object store with settings for a test environment. Only a single OSD is required in this example. -# kubectl create -f object-multisite.yaml +# Create an object store with multisite settings that is syncing with another multisite cluster for a +# production environment. A minimum of 3 hosts with OSDs are required in this example. +# kubectl create -f object-multisite-pull-realm.yaml ################################################################################################################# apiVersion: v1 kind: Secret metadata: name: realm-a-keys - namespace: myceph + namespace: rook-ceph-secondary data: # TODO: Replace with keys for your cluster # these keys should be the base64 encoded versions of the actual keys or copied from the realm's on the other cluster's secret @@ -17,7 +18,7 @@ apiVersion: ceph.rook.io/v1 kind: CephObjectRealm metadata: name: realm-a - namespace: new-rook-ceph-namespace + namespace: rook-ceph-secondary spec: # This endpoint in this section needs is an endpoint from the master zone in the master zone group of realm-a. See object-multisite.md for more details. pull: @@ -27,7 +28,7 @@ apiVersion: ceph.rook.io/v1 kind: CephObjectZoneGroup metadata: name: zonegroup-a - namespace: new-rook-ceph-namespace + namespace: rook-ceph-secondary spec: realm: realm-a --- @@ -35,25 +36,25 @@ apiVersion: ceph.rook.io/v1 kind: CephObjectZone metadata: name: zone-b - namespace: new-rook-ceph-namespace + namespace: rook-ceph-secondary spec: zoneGroup: zonegroup-a metadataPool: failureDomain: host replicated: - size: 1 + size: 3 requireSafeReplicaSize: true dataPool: failureDomain: host replicated: - size: 1 + size: 3 requireSafeReplicaSize: true --- apiVersion: ceph.rook.io/v1 kind: CephObjectStore metadata: name: zone-b-multisite-store - namespace: new-rook-ceph-namespace + namespace: rook-ceph-secondary spec: gateway: port: 80 diff --git a/cluster/examples/kubernetes/ceph/object-multisite-test.yaml b/cluster/examples/kubernetes/ceph/object-multisite-test.yaml new file mode 100644 index 000000000..edb68deaa --- /dev/null +++ b/cluster/examples/kubernetes/ceph/object-multisite-test.yaml @@ -0,0 +1,50 @@ +################################################################################################################# +# Create an object store with settings for a test environment. Only a single OSD is required in this example. +# kubectl create -f object-multisite-test.yaml +################################################################################################################# +apiVersion: ceph.rook.io/v1 +kind: CephObjectRealm +metadata: + name: realm-a + namespace: rook-ceph # namespace:cluster +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZoneGroup +metadata: + name: zonegroup-a + namespace: rook-ceph # namespace:cluster +spec: + realm: realm-a +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZone +metadata: + name: zone-a + namespace: rook-ceph # namespace:cluster +spec: + zoneGroup: zonegroup-a + metadataPool: + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false + dataPool: + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false + parameters: + compression_mode: none +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: multisite-store + namespace: rook-ceph # namespace:cluster +spec: + gateway: + port: 80 + # securePort: 443 + instances: 1 + zone: + name: zone-a diff --git a/cluster/examples/kubernetes/ceph/object-multisite.yaml b/cluster/examples/kubernetes/ceph/object-multisite.yaml index 247d825c4..20614fb6b 100644 --- a/cluster/examples/kubernetes/ceph/object-multisite.yaml +++ b/cluster/examples/kubernetes/ceph/object-multisite.yaml @@ -1,5 +1,6 @@ ################################################################################################################# -# Create an object store with settings for a test environment. Only a single OSD is required in this example. +# Create an object store with multisite settings for a production environment. A minimum of 3 hosts with OSDs +# are required in this example. # kubectl create -f object-multisite.yaml ################################################################################################################# apiVersion: ceph.rook.io/v1 @@ -26,12 +27,12 @@ spec: metadataPool: failureDomain: host replicated: - size: 1 + size: 3 requireSafeReplicaSize: true dataPool: failureDomain: host replicated: - size: 1 + size: 3 requireSafeReplicaSize: true parameters: compression_mode: none diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index e1faa7075..bc53f4e26 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -246,11 +246,57 @@ EOF kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d > "${DIR}"/"${SERVICE}".ca } +function deploy_first_rook_cluster() { + BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1) + cd cluster/examples/kubernetes/ceph/ + kubectl create -f crds.yaml -f common.yaml -f operator.yaml + yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false + yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false + yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}1 + kubectl create -f cluster-test.yaml -f toolbox.yaml +} + +function wait_for_rgw_pods() { + for i in {1..120}; do kubectl -n $1 get pod -l app=rook-ceph-rgw -o jsonpath='{.items[0].metadata.name}' && echo -e "\nrgw pods found" && break || echo -e "\nwaiting for rgw pods"; sleep 5; done +} + +function deploy_second_rook_cluster() { + BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1) + cd cluster/examples/kubernetes/ceph/ + NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f - + sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' cluster-test.yaml + yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}2 + yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external" + yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary + kubectl create -f cluster-test.yaml -f toolbox.yaml +} + +function write_object_to_cluster1_read_from_cluster2() { + cd cluster/examples/kubernetes/ceph/ + echo "[default]" > s3cfg + echo "host_bucket = no.way.in.hell" >> ./s3cfg + echo "use_https = False" >> ./s3cfg + fallocate -l 1M ./1M.dat + echo "hello world" >> ./1M.dat + CLUSTER_1_IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") + BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") + BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") + ACCESS_KEY=$(echo ${BASE64_ACCESS_KEY} | base64 --decode) + SECRET_KEY=$(echo ${BASE64_SECRET_KEY} | base64 --decode) + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_1_IP_ADDR} mb s3://bkt + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_1_IP_ADDR} put ./1M.dat s3://bkt + CLUSTER_2_IP_ADDR=$(kubectl -n rook-ceph-secondary get svc rook-ceph-rgw-zone-b-multisite-store -o jsonpath="{.spec.clusterIP}") + s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_2_IP_ADDR} get s3://bkt/1M.dat 1M-get.dat --force + diff 1M.dat 1M-get.dat +} + selected_function="$1" if [ "$selected_function" = "generate_tls_config" ]; then $selected_function $2 $3 $4 $5 elif [ "$selected_function" = "wait_for_ceph_to_be_ready" ]; then $selected_function $2 $3 +elif [ "$selected_function" = "wait_for_rgw_pods" ]; then + $selected_function $2 else $selected_function fi