ceph: rgw multisite intgration testing

This test:
- starts up 2 minikube clusters
- create 2 ceph clusters
- creates object multisite CRDs on each cluster and
syncs the clusters
- writes an object to cluster 1 and reads it on
cluster 2

This commit also adds new functions in
github-action-helper.sh that aid in the multisite
test and the multi cluster mirroring test.

Signed-off-by: Ali Maredia <amaredia@redhat.com>

.github/workflows/canary-integration-test.yml

@@ -720,26 +720,18 @@ jobs:
 
     - name: deploy first cluster rook
       run: |
-        BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1)
+        tests/scripts/github-action-helper.sh deploy_first_rook_cluster
         cd cluster/examples/kubernetes/ceph/
-        kubectl create -f crds.yaml -f common.yaml -f operator.yaml
-        yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false
-        yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false
-        yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}1
-        kubectl create -f cluster-test.yaml -f rbdmirror.yaml -f filesystem-mirror.yaml -f toolbox.yaml
+        kubectl create -f rbdmirror.yaml -f filesystem-mirror.yaml
 
     # cephfs-mirroring is a push operation
     # running bootstrap create on secondary and bootstrap import on primary. mirror daemons on primary.
     - name: deploy second cluster rook
       run: |
-        BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1)
+        tests/scripts/github-action-helper.sh deploy_second_rook_cluster
         cd cluster/examples/kubernetes/ceph/
-        NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f -
-        sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' cluster-test.yaml rbdmirror.yaml
-        yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}2
-        yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-secondary"
-        yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary
-        kubectl create -f cluster-test.yaml -f rbdmirror.yaml -f toolbox.yaml
+        sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' rbdmirror.yaml
+        kubectl create -f rbdmirror.yaml
 
     - name: wait for ceph cluster 1 to be ready
       run: |
@@ -876,3 +868,85 @@ jobs:
       if: failure()
       uses: mxschmitt/action-tmate@v3
       timeout-minutes: 60
+
+  rgw-multisite-testing:
+    runs-on: ubuntu-18.04
+    if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
+    steps:
+    - name: checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: setup golang
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16
+
+    - name: install deps
+      run: |
+        tests/scripts/github-action-helper.sh install_deps
+        sudo apt-get install -y s3cmd
+
+    - name: setup minikube
+      uses: manusa/actions-setup-minikube@v2.3.1
+      with:
+        minikube version: 'v1.18.1'
+        kubernetes version: 'v1.19.2'
+        start args: --memory 6g --cpus=2
+        github token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: use local disk into two partitions
+      run: |
+        BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
+        tests/scripts/github-action-helper.sh use_local_disk
+        tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2
+        sudo lsblk
+
+    - name: build rook
+      run: tests/scripts/github-action-helper.sh build_rook
+
+    - name: deploy first cluster rook
+      run: |
+        tests/scripts/github-action-helper.sh deploy_first_rook_cluster
+        kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml
+        # wait for multisite store to be created
+        tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph
+
+    - name: prep second cluster pull realm config
+      run: |
+        cd cluster/examples/kubernetes/ceph/
+        IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
+        yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80
+        BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
+        BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}")
+        sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml
+        sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml
+
+    - name: deploy second cluster rook
+      run: |
+        tests/scripts/github-action-helper.sh deploy_second_rook_cluster
+        kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml
+        # wait for realms to be pulled and zone-b-multisite-store to be created
+        tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary
+
+    - name: wait for ceph cluster 1 to be ready
+      run: |
+        mkdir test
+        tests/scripts/validate_cluster.sh osd 1
+        kubectl -n rook-ceph get pods
+
+    - name: write an object to one cluster, read from the other
+      run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2
+
+    - name: upload test result
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: rgw-multisite-testing
+        path: test
+
+    - name: setup tmate session for debugging
+      if: failure()
+      uses: mxschmitt/action-tmate@v3
+      timeout-minutes: 60

cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml

@@ -0,0 +1,63 @@
+#################################################################################################################
+# Create an object store with settings for a test environment. Only a single OSD is required in this example.
+#  kubectl create -f object-multisite-pull-realm-test.yaml
+#################################################################################################################
+apiVersion: v1
+kind: Secret
+metadata:
+  name: realm-a-keys
+  namespace: rook-ceph-secondary
+data:
+  # TODO: Replace with keys for your cluster
+  # these keys should be the base64 encoded versions of the actual keys or copied from the realm's on the other cluster's secret
+  access-key: VzFjNFltMVdWRTFJWWxZelZWQT0=
+  secret-key: WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectRealm
+metadata:
+  name: realm-a
+  namespace: rook-ceph-secondary
+spec:
+  # This endpoint in this section needs is an endpoint from the master zone in the master zone group of realm-a. See object-multisite.md for more details.
+  pull:
+    endpoint: http://10.103.133.16:80
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZoneGroup
+metadata:
+  name: zonegroup-a
+  namespace: rook-ceph-secondary
+spec:
+  realm: realm-a
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZone
+metadata:
+  name: zone-b
+  namespace: rook-ceph-secondary
+spec:
+  zoneGroup: zonegroup-a
+  metadataPool:
+    failureDomain: host
+    replicated:
+      size: 1
+      requireSafeReplicaSize: false
+  dataPool:
+    failureDomain: host
+    replicated:
+      size: 1
+      requireSafeReplicaSize: false
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: zone-b-multisite-store
+  namespace: rook-ceph-secondary
+spec:
+  gateway:
+    port: 80
+    # securePort: 443
+    instances: 1
+  zone:
+    name: zone-b

cluster/examples/kubernetes/ceph/object-multisite-pull-realm.yaml

@@ -1,12 +1,13 @@
 #################################################################################################################
-# Create an object store with settings for a test environment. Only a single OSD is required in this example.
-#  kubectl create -f object-multisite.yaml
+# Create an object store with multisite settings that is syncing with another multisite cluster for a
+# production environment. A minimum of 3 hosts with OSDs are required in this example.
+#  kubectl create -f object-multisite-pull-realm.yaml
 #################################################################################################################
 apiVersion: v1
 kind: Secret
 metadata:
   name: realm-a-keys
-  namespace: myceph
+  namespace: rook-ceph-secondary
 data:
   # TODO: Replace with keys for your cluster
   # these keys should be the base64 encoded versions of the actual keys or copied from the realm's on the other cluster's secret
@@ -17,7 +18,7 @@ apiVersion: ceph.rook.io/v1
 kind: CephObjectRealm
 metadata:
   name: realm-a
-  namespace: new-rook-ceph-namespace
+  namespace: rook-ceph-secondary
 spec:
   # This endpoint in this section needs is an endpoint from the master zone in the master zone group of realm-a. See object-multisite.md for more details.
   pull:
@@ -27,7 +28,7 @@ apiVersion: ceph.rook.io/v1
 kind: CephObjectZoneGroup
 metadata:
   name: zonegroup-a
-  namespace: new-rook-ceph-namespace
+  namespace: rook-ceph-secondary
 spec:
   realm: realm-a
 ---
@@ -35,25 +36,25 @@ apiVersion: ceph.rook.io/v1
 kind: CephObjectZone
 metadata:
   name: zone-b
-  namespace: new-rook-ceph-namespace
+  namespace: rook-ceph-secondary
 spec:
   zoneGroup: zonegroup-a
   metadataPool:
     failureDomain: host
     replicated:
-      size: 1
+      size: 3
       requireSafeReplicaSize: true
   dataPool:
     failureDomain: host
     replicated:
-      size: 1
+      size: 3
       requireSafeReplicaSize: true
 ---
 apiVersion: ceph.rook.io/v1
 kind: CephObjectStore
 metadata:
   name: zone-b-multisite-store
-  namespace: new-rook-ceph-namespace
+  namespace: rook-ceph-secondary
 spec:
   gateway:
     port: 80

cluster/examples/kubernetes/ceph/object-multisite-test.yaml

@@ -0,0 +1,50 @@
+#################################################################################################################
+# Create an object store with settings for a test environment. Only a single OSD is required in this example.
+#  kubectl create -f object-multisite-test.yaml
+#################################################################################################################
+apiVersion: ceph.rook.io/v1
+kind: CephObjectRealm
+metadata:
+  name: realm-a
+  namespace: rook-ceph # namespace:cluster
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZoneGroup
+metadata:
+  name: zonegroup-a
+  namespace: rook-ceph # namespace:cluster
+spec:
+  realm: realm-a
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZone
+metadata:
+  name: zone-a
+  namespace: rook-ceph # namespace:cluster
+spec:
+  zoneGroup: zonegroup-a
+  metadataPool:
+    failureDomain: host
+    replicated:
+      size: 1
+      requireSafeReplicaSize: false
+  dataPool:
+    failureDomain: host
+    replicated:
+      size: 1
+      requireSafeReplicaSize: false
+    parameters:
+      compression_mode: none
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: multisite-store
+  namespace: rook-ceph # namespace:cluster
+spec:
+  gateway:
+    port: 80
+    # securePort: 443
+    instances: 1
+  zone:
+    name: zone-a

cluster/examples/kubernetes/ceph/object-multisite.yaml

@@ -1,5 +1,6 @@
 #################################################################################################################
-# Create an object store with settings for a test environment. Only a single OSD is required in this example.
+# Create an object store with multisite settings for a production environment. A minimum of 3 hosts with OSDs
+# are required in this example.
 #  kubectl create -f object-multisite.yaml
 #################################################################################################################
 apiVersion: ceph.rook.io/v1
@@ -26,12 +27,12 @@ spec:
   metadataPool:
     failureDomain: host
     replicated:
-      size: 1
+      size: 3
       requireSafeReplicaSize: true
   dataPool:
     failureDomain: host
     replicated:
-      size: 1
+      size: 3
       requireSafeReplicaSize: true
     parameters:
       compression_mode: none

tests/scripts/github-action-helper.sh

@@ -246,11 +246,57 @@ EOF
   kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d > "${DIR}"/"${SERVICE}".ca
 }
 
+function deploy_first_rook_cluster() {
+  BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1)
+  cd cluster/examples/kubernetes/ceph/
+  kubectl create -f crds.yaml -f common.yaml -f operator.yaml
+  yq w -i -d1 cluster-test.yaml spec.dashboard.enabled false
+  yq w -i -d1 cluster-test.yaml spec.storage.useAllDevices false
+  yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}1
+  kubectl create -f cluster-test.yaml -f toolbox.yaml
+}
+
+function wait_for_rgw_pods() {
+  for i in {1..120}; do kubectl -n $1 get pod -l app=rook-ceph-rgw -o jsonpath='{.items[0].metadata.name}' && echo -e "\nrgw pods found" && break || echo -e "\nwaiting for rgw pods"; sleep 5; done
+}
+
+function deploy_second_rook_cluster() {
+  BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1)
+  cd cluster/examples/kubernetes/ceph/
+  NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f -
+  sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' cluster-test.yaml
+  yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter ${BLOCK}2
+  yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external"
+  yq w -i toolbox.yaml metadata.namespace rook-ceph-secondary
+  kubectl create -f cluster-test.yaml -f toolbox.yaml
+}
+
+function write_object_to_cluster1_read_from_cluster2() {
+  cd cluster/examples/kubernetes/ceph/
+  echo "[default]" > s3cfg
+  echo "host_bucket = no.way.in.hell" >> ./s3cfg
+  echo "use_https = False" >> ./s3cfg
+  fallocate -l 1M ./1M.dat
+  echo "hello world" >> ./1M.dat
+  CLUSTER_1_IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
+  BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
+  BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}")
+  ACCESS_KEY=$(echo ${BASE64_ACCESS_KEY} | base64 --decode)
+  SECRET_KEY=$(echo ${BASE64_SECRET_KEY} | base64 --decode)
+  s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_1_IP_ADDR} mb s3://bkt
+  s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_1_IP_ADDR} put ./1M.dat s3://bkt
+  CLUSTER_2_IP_ADDR=$(kubectl -n rook-ceph-secondary get svc rook-ceph-rgw-zone-b-multisite-store -o jsonpath="{.spec.clusterIP}")
+  s3cmd --config=s3cfg --access_key=${ACCESS_KEY} --secret_key=${SECRET_KEY} --host=${CLUSTER_2_IP_ADDR} get s3://bkt/1M.dat 1M-get.dat --force
+  diff 1M.dat 1M-get.dat
+}
+
 selected_function="$1"
 if [ "$selected_function" = "generate_tls_config" ]; then
     $selected_function $2 $3 $4 $5
 elif [ "$selected_function" = "wait_for_ceph_to_be_ready" ]; then
      $selected_function $2 $3
+elif [ "$selected_function" = "wait_for_rgw_pods" ]; then
+     $selected_function $2
 else
   $selected_function
 fi