diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index bfe834bdc..824c7ce3e 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -70,6 +70,234 @@ jobs: # copy the test file # execute the test file + pvc: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: setup golang + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: install deps + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + - name: setup minikube + uses: manusa/actions-setup-minikube@v2.0.1 + with: + minikube version: 'v1.13.1' + kubernetes version: 'v1.19.2' + start args: --memory 6g --cpus=2 + + - name: use local disk + run: | + BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1 + sudo dmsetup version||true + sudo swapoff --all --verbose + sudo umount /mnt + # search for the device since it keeps changing between sda and sdb + sudo wipefs --all --force "$BLOCK_DATA_PART" + sudo lsblk + + - name: check k8s cluster status + run: | + kubectl cluster-info + kubectl get pods -n kube-system + + - name: build rook + run: | + # set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version" + GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build + docker images + docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master + + - name: rook prerequisites + run: | + BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/localPathPV.sh "$BLOCK" + kubectl create -f cluster/examples/kubernetes/ceph/common.yaml + + - name: deploy rook + run: | + kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml + yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false + kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml + + - name: wait for prepare pod + run: | + timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true + timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true + kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true + kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true + + - name: wait for ceph to be ready + run: | + tests/scripts/validate_cluster.sh osd + kubectl -n rook-ceph get pods + + pvc-db: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: setup golang + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: install deps + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + sudo apt-get install -y gdisk + + - name: setup minikube + uses: manusa/actions-setup-minikube@v2.0.1 + with: + minikube version: 'v1.13.1' + kubernetes version: 'v1.19.2' + start args: --memory 6g --cpus=2 + + - name: use local disk + run: | + BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1 + sudo dmsetup version||true + sudo swapoff --all --verbose + sudo umount /mnt + # search for the device since it keeps changing between sda and sdb + sudo wipefs --all --force "$BLOCK_DATA_PART" + sudo lsblk + + - name: create bluestore partitions and PVCs + run: | + BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK_PART="$BLOCK"2 + DB_PART="$BLOCK"1 + tests/scripts/create-bluestore-partitions.sh "$BLOCK" block.db + tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART" + + - name: check k8s cluster status + run: | + kubectl cluster-info + kubectl get pods -n kube-system + + - name: build rook + run: | + # set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version" + GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build + docker images + docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master + + - name: rook prerequisites + run: | + kubectl create -f cluster/examples/kubernetes/ceph/common.yaml + + - name: deploy rook + run: | + kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml + yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false + cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml + + - name: wait for prepare pod + run: | + timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true + timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true + kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true + kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true + + - name: wait for ceph to be ready + run: | + tests/scripts/validate_cluster.sh osd + kubectl -n rook-ceph get pods + + pvc-db-wal: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: setup golang + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: install deps + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + sudo apt-get install -y gdisk + + - name: setup minikube + uses: manusa/actions-setup-minikube@v2.0.1 + with: + minikube version: 'v1.13.1' + kubernetes version: 'v1.19.2' + start args: --memory 6g --cpus=2 + + - name: use local disk + run: | + BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1 + sudo dmsetup version||true + sudo swapoff --all --verbose + sudo umount /mnt + # search for the device since it keeps changing between sda and sdb + sudo wipefs --all --force "$BLOCK_DATA_PART" + sudo lsblk + + - name: create bluestore partitions and PVCs + run: | + BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK_PART="$BLOCK"3 + DB_PART="$BLOCK"1 + WAL_PART="$BLOCK"2 + tests/scripts/create-bluestore-partitions.sh "$BLOCK" block.wal + tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART" "$WAL_PART" + + - name: check k8s cluster status + run: | + kubectl cluster-info + kubectl get pods -n kube-system + + - name: build rook + run: | + # set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version" + GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build + docker images + docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master + + - name: rook prerequisites + run: | + kubectl create -f cluster/examples/kubernetes/ceph/common.yaml + + - name: deploy rook + run: | + kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml + yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false + cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml + cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml + + - name: wait for prepare pod + run: | + timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true + timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true + kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true + kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true + + - name: wait for ceph to be ready + run: | + tests/scripts/validate_cluster.sh osd + kubectl -n rook-ceph get pods + encryption-pvc: runs-on: ubuntu-latest steps: @@ -138,6 +366,164 @@ jobs: kubectl -n rook-ceph get secrets sudo lsblk + encryption-pvc-db: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: setup golang + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: install deps + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + sudo apt-get install -y gdisk + + - name: setup minikube + uses: manusa/actions-setup-minikube@v2.0.1 + with: + minikube version: 'v1.13.1' + kubernetes version: 'v1.19.2' + start args: --memory 6g --cpus=2 + + - name: use local disk + run: | + BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1 + sudo dmsetup version||true + sudo swapoff --all --verbose + sudo umount /mnt + # search for the device since it keeps changing between sda and sdb + sudo wipefs --all --force "$BLOCK_DATA_PART" + sudo lsblk + + - name: create bluestore partitions and PVCs + run: | + BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK_PART="$BLOCK"3 + DB_PART="$BLOCK"1 + WAL_PART="$BLOCK"2 + tests/scripts/create-bluestore-partitions.sh "$BLOCK" block.wal + tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART" "$WAL_PART" + + - name: check k8s cluster status + run: | + kubectl cluster-info + kubectl get pods -n kube-system + + - name: build rook + run: | + # set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version" + GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build + docker images + docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master + + - name: rook prerequisites + run: | + kubectl create -f cluster/examples/kubernetes/ceph/common.yaml + + - name: deploy rook + run: | + kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml + cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml + cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml + + - name: wait for prepare pod + run: | + timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true + timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true + kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true + kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true + + - name: wait for ceph to be ready + run: | + tests/scripts/validate_cluster.sh osd + kubectl -n rook-ceph get pods + kubectl -n rook-ceph get secrets + + encryption-pvc-db-wal: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: setup golang + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: install deps + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + sudo apt-get install -y gdisk + + - name: setup minikube + uses: manusa/actions-setup-minikube@v2.0.1 + with: + minikube version: 'v1.13.1' + kubernetes version: 'v1.19.2' + start args: --memory 6g --cpus=2 + + - name: use local disk + run: | + BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1 + sudo dmsetup version||true + sudo swapoff --all --verbose + sudo umount /mnt + # search for the device since it keeps changing between sda and sdb + sudo wipefs --all --force "$BLOCK_DATA_PART" + sudo lsblk + + - name: create bluestore partitions and PVCs + run: | + BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK_PART="$BLOCK"2 + DB_PART="$BLOCK"1 + tests/scripts/create-bluestore-partitions.sh "$BLOCK" block.db + tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART" + + - name: check k8s cluster status + run: | + kubectl cluster-info + kubectl get pods -n kube-system + + - name: build rook + run: | + # set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version" + GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build + docker images + docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master + + - name: rook prerequisites + run: | + kubectl create -f cluster/examples/kubernetes/ceph/common.yaml + + - name: deploy rook + run: | + kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml + cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml + + - name: wait for prepare pod + run: | + timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true + timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true + kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true + kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true + + - name: wait for ceph to be ready + run: | + tests/scripts/validate_cluster.sh osd + kubectl -n rook-ceph get pods + kubectl -n rook-ceph get secrets + encryption-pvc-kms-vault-token-auth: runs-on: ubuntu-latest steps: diff --git a/tests/manifests/test-cluster-on-pvc-encrypted.yaml b/tests/manifests/test-cluster-on-pvc-encrypted.yaml index 41b2bbb1a..754ddf57a 100644 --- a/tests/manifests/test-cluster-on-pvc-encrypted.yaml +++ b/tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1,4 +1,3 @@ ---- apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -24,19 +23,19 @@ spec: disable: true storage: storageClassDeviceSets: - - name: set1 - count: 1 - portable: false - tuneDeviceClass: false - encrypted: true - volumeClaimTemplates: - - metadata: - name: data - spec: - resources: - requests: - storage: 10Gi - storageClassName: manual - volumeMode: Block - accessModes: - - ReadWriteOnce + - name: set1 + count: 1 + portable: false + tuneDeviceClass: false + encrypted: true + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + storageClassName: manual + volumeMode: Block + accessModes: + - ReadWriteOnce diff --git a/tests/manifests/test-on-pvc-db.yaml b/tests/manifests/test-on-pvc-db.yaml new file mode 100644 index 000000000..0a7a569ed --- /dev/null +++ b/tests/manifests/test-on-pvc-db.yaml @@ -0,0 +1,10 @@ + - metadata: + name: metadata + spec: + resources: + requests: + storage: 2Gi + storageClassName: manual + volumeMode: Block + accessModes: + - ReadWriteOnce diff --git a/tests/manifests/test-on-pvc-wal.yaml b/tests/manifests/test-on-pvc-wal.yaml new file mode 100644 index 000000000..2b91a2ebc --- /dev/null +++ b/tests/manifests/test-on-pvc-wal.yaml @@ -0,0 +1,10 @@ + - metadata: + name: wal + spec: + resources: + requests: + storage: 2Gi + storageClassName: manual + volumeMode: Block + accessModes: + - ReadWriteOnce diff --git a/tests/scripts/create-bluestore-partitions.sh b/tests/scripts/create-bluestore-partitions.sh new file mode 100755 index 000000000..8c9263cb9 --- /dev/null +++ b/tests/scripts/create-bluestore-partitions.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -ex + +############# +# VARIABLES # +############# + +: "${BLUESTORE_TYPE:=${2}}" +: "${DISK:=${1}}" +SIZE=2048M + +############# +# FUNCTIONS # +############# + +function wipe_disk { + sudo sgdisk --zap-all --clear --mbrtogpt -g -- "$DISK" + sudo dd if=/dev/zero of="$DISK" bs=1M count=10 + sudo parted -s "$DISK" mklabel gpt + sudo partprobe "$DISK" + sudo udevadm settle + sudo parted "$DISK" -s print +} + +function create_partition { + sudo sgdisk --new=0:0:+"$SIZE" --change-name=0:"$1" --mbrtogpt -- "$DISK" +} + +function create_block_partition { + sudo sgdisk --largest-new=0 --change-name=0:'block' --mbrtogpt -- "$DISK" +} + +######## +# MAIN # +######## + +# First wipe the disk +wipe_disk + +case "$BLUESTORE_TYPE" in + block.db) + create_partition block.db + ;; + block.wal) + create_partition block.db + create_partition block.wal + ;; + *) + echo "invalid bluestore configuration $BLUESTORE_TYPE" >&2 + exit 1 + esac + +# Create final block partitions +create_block_partition + +# Inform the kernel of partition table changes +sudo partprobe + +# Print drives +sudo lsblk +sudo parted "$DISK" -s print \ No newline at end of file diff --git a/tests/scripts/localPathPV.sh b/tests/scripts/localPathPV.sh index 874a9dc5b..f40a9c2c4 100755 --- a/tests/scripts/localPathPV.sh +++ b/tests/scripts/localPathPV.sh @@ -6,6 +6,9 @@ if [ $# -ge 1 ] ; then test_scratch_device=$1 fi +db_device=$2 +wal_device=$3 + lsblk if [ ! -b "${test_scratch_device}" ] ; then @@ -129,4 +132,81 @@ metadata: name: manual provisioner: kubernetes.io/no-provisioner volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete eof + +function add_db_pvc { +cat <