ci: add more osd on pvc scenario

Add the following scenario:

* simple osd on pvc
* osd on pvc with db device
* osd on pvc with wal device
* encrypted simple osd on pvc
* encrypted osd on pvc with db device
* encrypted osd on pvc with wal device

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2020-10-30 16:17:10 +01:00
1 parent ea1d71cbfb
commit 3403776c3e
7 files changed
+572 -18

No files matched your search

@@ -70,6 +70,234 @@ jobs:
# copy the test file
# execute the test file
pvc:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
- name: setup golang
uses: actions/setup-go@v2
with:
go-version: 1.15
- name: install deps
run: |
sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq
sudo chmod +x /usr/bin/yq
- name: setup minikube
uses: manusa/actions-setup-minikube@v2.0.1
with:
minikube version: 'v1.13.1'
kubernetes version: 'v1.19.2'
start args: --memory 6g --cpus=2
- name: use local disk
run: |
BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1
sudo dmsetup version||true
sudo swapoff --all --verbose
sudo umount /mnt
# search for the device since it keeps changing between sda and sdb
sudo wipefs --all --force "$BLOCK_DATA_PART"
sudo lsblk
- name: check k8s cluster status
run: |
kubectl cluster-info
kubectl get pods -n kube-system
- name: build rook
run: |
# set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version"
GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build
docker images
docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master
- name: rook prerequisites
run: |
BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
tests/scripts/localPathPV.sh "$BLOCK"
kubectl create -f cluster/examples/kubernetes/ceph/common.yaml
- name: deploy rook
run: |
kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml
- name: wait for prepare pod
run: |
timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true
timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true
kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true
kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true
- name: wait for ceph to be ready
run: |
tests/scripts/validate_cluster.sh osd
kubectl -n rook-ceph get pods
pvc-db:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
- name: setup golang
uses: actions/setup-go@v2
with:
go-version: 1.15
- name: install deps
run: |
sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq
sudo chmod +x /usr/bin/yq
sudo apt-get install -y gdisk
- name: setup minikube
uses: manusa/actions-setup-minikube@v2.0.1
with:
minikube version: 'v1.13.1'
kubernetes version: 'v1.19.2'
start args: --memory 6g --cpus=2
- name: use local disk
run: |
BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1
sudo dmsetup version||true
sudo swapoff --all --verbose
sudo umount /mnt
# search for the device since it keeps changing between sda and sdb
sudo wipefs --all --force "$BLOCK_DATA_PART"
sudo lsblk
- name: create bluestore partitions and PVCs
run: |
BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
BLOCK_PART="$BLOCK"2
DB_PART="$BLOCK"1
tests/scripts/create-bluestore-partitions.sh "$BLOCK" block.db
tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART"
- name: check k8s cluster status
run: |
kubectl cluster-info
kubectl get pods -n kube-system
- name: build rook
run: |
# set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version"
GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build
docker images
docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master
- name: rook prerequisites
run: |
kubectl create -f cluster/examples/kubernetes/ceph/common.yaml
- name: deploy rook
run: |
kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml
- name: wait for prepare pod
run: |
timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true
timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true
kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true
kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true
- name: wait for ceph to be ready
run: |
tests/scripts/validate_cluster.sh osd
kubectl -n rook-ceph get pods
pvc-db-wal:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
- name: setup golang
uses: actions/setup-go@v2
with:
go-version: 1.15
- name: install deps
run: |
sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq
sudo chmod +x /usr/bin/yq
sudo apt-get install -y gdisk
- name: setup minikube
uses: manusa/actions-setup-minikube@v2.0.1
with:
minikube version: 'v1.13.1'
kubernetes version: 'v1.19.2'
start args: --memory 6g --cpus=2
- name: use local disk
run: |
BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1
sudo dmsetup version||true
sudo swapoff --all --verbose
sudo umount /mnt
# search for the device since it keeps changing between sda and sdb
sudo wipefs --all --force "$BLOCK_DATA_PART"
sudo lsblk
- name: create bluestore partitions and PVCs
run: |
BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
BLOCK_PART="$BLOCK"3
DB_PART="$BLOCK"1
WAL_PART="$BLOCK"2
tests/scripts/create-bluestore-partitions.sh "$BLOCK" block.wal
tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART" "$WAL_PART"
- name: check k8s cluster status
run: |
kubectl cluster-info
kubectl get pods -n kube-system
- name: build rook
run: |
# set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version"
GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build
docker images
docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master
- name: rook prerequisites
run: |
kubectl create -f cluster/examples/kubernetes/ceph/common.yaml
- name: deploy rook
run: |
kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml
- name: wait for prepare pod
run: |
timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true
timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true
kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true
kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true
- name: wait for ceph to be ready
run: |
tests/scripts/validate_cluster.sh osd
kubectl -n rook-ceph get pods
encryption-pvc:
runs-on: ubuntu-latest
steps:
@@ -138,6 +366,164 @@ jobs:
kubectl -n rook-ceph get secrets
sudo lsblk
encryption-pvc-db:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
- name: setup golang
uses: actions/setup-go@v2
with:
go-version: 1.15
- name: install deps
run: |
sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq
sudo chmod +x /usr/bin/yq
sudo apt-get install -y gdisk
- name: setup minikube
uses: manusa/actions-setup-minikube@v2.0.1
with:
minikube version: 'v1.13.1'
kubernetes version: 'v1.19.2'
start args: --memory 6g --cpus=2
- name: use local disk
run: |
BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1
sudo dmsetup version||true
sudo swapoff --all --verbose
sudo umount /mnt
# search for the device since it keeps changing between sda and sdb
sudo wipefs --all --force "$BLOCK_DATA_PART"
sudo lsblk
- name: create bluestore partitions and PVCs
run: |
BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
BLOCK_PART="$BLOCK"3
DB_PART="$BLOCK"1
WAL_PART="$BLOCK"2
tests/scripts/create-bluestore-partitions.sh "$BLOCK" block.wal
tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART" "$WAL_PART"
- name: check k8s cluster status
run: |
kubectl cluster-info
kubectl get pods -n kube-system
- name: build rook
run: |
# set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version"
GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build
docker images
docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master
- name: rook prerequisites
run: |
kubectl create -f cluster/examples/kubernetes/ceph/common.yaml
- name: deploy rook
run: |
kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml
- name: wait for prepare pod
run: |
timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true
timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true
kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true
kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true
- name: wait for ceph to be ready
run: |
tests/scripts/validate_cluster.sh osd
kubectl -n rook-ceph get pods
kubectl -n rook-ceph get secrets
encryption-pvc-db-wal:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
- name: setup golang
uses: actions/setup-go@v2
with:
go-version: 1.15
- name: install deps
run: |
sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq
sudo chmod +x /usr/bin/yq
sudo apt-get install -y gdisk
- name: setup minikube
uses: manusa/actions-setup-minikube@v2.0.1
with:
minikube version: 'v1.13.1'
kubernetes version: 'v1.19.2'
start args: --memory 6g --cpus=2
- name: use local disk
run: |
BLOCK_DATA_PART=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1
sudo dmsetup version||true
sudo swapoff --all --verbose
sudo umount /mnt
# search for the device since it keeps changing between sda and sdb
sudo wipefs --all --force "$BLOCK_DATA_PART"
sudo lsblk
- name: create bluestore partitions and PVCs
run: |
BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)
BLOCK_PART="$BLOCK"2
DB_PART="$BLOCK"1
tests/scripts/create-bluestore-partitions.sh "$BLOCK" block.db
tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART"
- name: check k8s cluster status
run: |
kubectl cluster-info
kubectl get pods -n kube-system
- name: build rook
run: |
# set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version"
GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' VERSION=0 build
docker images
docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:master
- name: rook prerequisites
run: |
kubectl create -f cluster/examples/kubernetes/ceph/common.yaml
- name: deploy rook
run: |
kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml
- name: wait for prepare pod
run: |
timeout 180 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done'||true
timeout 60 sh -c 'until kubectl -n rook-ceph logs $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd,ceph_daemon_id=0 -o jsonpath='{.items[*].metadata.name}') --all-containers; do echo "waiting for osd container" && sleep 1; done'||true
kubectl -n rook-ceph describe job/$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')||true
kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true
- name: wait for ceph to be ready
run: |
tests/scripts/validate_cluster.sh osd
kubectl -n rook-ceph get pods
kubectl -n rook-ceph get secrets
encryption-pvc-kms-vault-token-auth:
runs-on: ubuntu-latest
steps:
@@ -1,4 +1,3 @@
---
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
@@ -24,19 +23,19 @@ spec:
disable: true
storage:
storageClassDeviceSets:
- name: set1
count: 1
portable: false
tuneDeviceClass: false
encrypted: true
volumeClaimTemplates:
- metadata:
name: data
spec:
resources:
requests:
storage: 10Gi
storageClassName: manual
volumeMode: Block
accessModes:
- ReadWriteOnce
- name: set1
count: 1
portable: false
tuneDeviceClass: false
encrypted: true
volumeClaimTemplates:
- metadata:
name: data
spec:
resources:
requests:
storage: 10Gi
storageClassName: manual
volumeMode: Block
accessModes:
- ReadWriteOnce
+10
View File
@@ -0,0 +1,10 @@
- metadata:
name: metadata
spec:
resources:
requests:
storage: 2Gi
storageClassName: manual
volumeMode: Block
accessModes:
- ReadWriteOnce
+10
View File
@@ -0,0 +1,10 @@
- metadata:
name: wal
spec:
resources:
requests:
storage: 2Gi
storageClassName: manual
volumeMode: Block
accessModes:
- ReadWriteOnce
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
set -ex
#############
# VARIABLES #
#############
: "${BLUESTORE_TYPE:=${2}}"
: "${DISK:=${1}}"
SIZE=2048M
#############
# FUNCTIONS #
#############
function wipe_disk {
sudo sgdisk --zap-all --clear --mbrtogpt -g -- "$DISK"
sudo dd if=/dev/zero of="$DISK" bs=1M count=10
sudo parted -s "$DISK" mklabel gpt
sudo partprobe "$DISK"
sudo udevadm settle
sudo parted "$DISK" -s print
}
function create_partition {
sudo sgdisk --new=0:0:+"$SIZE" --change-name=0:"$1" --mbrtogpt -- "$DISK"
}
function create_block_partition {
sudo sgdisk --largest-new=0 --change-name=0:'block' --mbrtogpt -- "$DISK"
}
########
# MAIN #
########
# First wipe the disk
wipe_disk
case "$BLUESTORE_TYPE" in
block.db)
create_partition block.db
;;
block.wal)
create_partition block.db
create_partition block.wal
;;
*)
echo "invalid bluestore configuration $BLUESTORE_TYPE" >&2
exit 1
esac
# Create final block partitions
create_block_partition
# Inform the kernel of partition table changes
sudo partprobe
# Print drives
sudo lsblk
sudo parted "$DISK" -s print
+80
View File
@@ -6,6 +6,9 @@ if [ $# -ge 1 ] ; then
test_scratch_device=$1
fi
db_device=$2
wal_device=$3
lsblk
if [ ! -b "${test_scratch_device}" ] ; then
@@ -129,4 +132,81 @@ metadata:
name: manual
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Delete
eof
function add_db_pvc {
cat <<eof | kubectl apply -f -
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: local-vol5
labels:
type: local
spec:
storageClassName: manual
capacity:
storage: 2Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
volumeMode: Block
local:
path: "${db_device}"
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: rook.io/has-disk
operator: In
values:
- "true"
eof
}
function add_wal_pvc {
cat <<eof | kubectl apply -f -
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: local-vol6
labels:
type: local
spec:
storageClassName: manual
capacity:
storage: 2Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
volumeMode: Block
local:
path: "${wal_device}"
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: rook.io/has-disk
operator: In
values:
- "true"
eof
}
########
# MAIN #
########
# Add a db device if needed
if [ -n "$db_device" ]; then
add_db_pvc
fi
# Add a wal device if needed
if [ -n "$wal_device" ]; then
add_wal_pvc
fi
kubectl get pv
+9 -1
View File
@@ -84,6 +84,12 @@ function display_status {
kubectl -n rook-ceph logs "$(kubectl -n rook-ceph -l app=rook-ceph-operator get pods -o jsonpath='{.items[*].metadata.name}')"
kubectl -n rook-ceph get pods
kubectl -n rook-ceph describe job/"$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}')"||true
kubectl -n rook-ceph describe deploy/rook-ceph-osd-0||true
kubectl get all -n rook-ceph -o wide
kubectl get all -n rook-ceph -o yaml
kubectl -n rook-ceph get cephcluster -o yaml
sudo lsblk
set -x
exit 1
@@ -139,4 +145,6 @@ done
echo "Ceph is up and running, have a look!"
$EXEC_COMMAND -s
kubectl -n rook-ceph get pods
kubectl -n rook-ceph get pods
kubectl -n rook-ceph logs "$(kubectl -n rook-ceph -l app=rook-ceph-operator get pods -o jsonpath='{.items[*].metadata.name}')"
kubectl -n rook-ceph get cephcluster -o yaml