forked from rook/rook
Oded Viner
2ebb5d968a
adds a new nvmeof minikube canary job for ceph v20. the test deploys rook with csi operator disabled for nvmeof flow. it validates pvc and pod io, gateway restart, and data persistence. Signed-off-by: Oded Viner <oviner@redhat.com>
2226 lines
113 KiB
YAML
2226 lines
113 KiB
YAML
name: Reusable canary integration tests
|
|
on:
|
|
# ONLY on.workflow_call ; call this from other files if needed
|
|
workflow_call:
|
|
inputs:
|
|
ceph_images:
|
|
description: "JSON list of Ceph images for creating Ceph cluster"
|
|
default: '["quay.io/ceph/ceph:v19"]'
|
|
type: string
|
|
kubernetes-version:
|
|
description: kubernetes version to use for the workflow
|
|
default: '["v1.35.0"]'
|
|
type: string
|
|
|
|
defaults:
|
|
run:
|
|
# reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
canary:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: setup csi-addons
|
|
run: tests/scripts/csiaddons.sh setup_csiaddons
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready all 2
|
|
|
|
- name: wait for ceph mgr to be ready
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr|grep -Eosq \"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\" ; do sleep 1 && echo 'waiting for the manager IP to be available'; done"
|
|
mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr)
|
|
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done"
|
|
|
|
- name: test osd.0 auth recovery from keyring file
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
osd_id=0
|
|
osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
if [ $osd_pod ]; then
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth del osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be deleted'; done";
|
|
kubectl -n rook-ceph delete pod $osd_pod;
|
|
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth get osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be recovered'; done";
|
|
osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}');
|
|
kubectl -n rook-ceph wait --for=condition=Ready pod/$osd_pod --timeout=120s;
|
|
else
|
|
echo "osd $osd_id not found, skipping test";
|
|
fi
|
|
|
|
- name: test external script create-external-cluster-resources.py
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- mkdir -p /etc/ceph/test-data
|
|
kubectl -n rook-ceph cp tests/external-cluster/ceph-status-out $toolbox:/etc/ceph/test-data/
|
|
kubectl -n rook-ceph cp tests/external-cluster/external-config.ini $toolbox:/etc/ceph/
|
|
kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources.py $toolbox:/etc/ceph
|
|
kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources-tests.py $toolbox:/etc/ceph
|
|
timeout 10 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool; do echo 'waiting for script to succeed' && sleep 1; done"
|
|
# print existing client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth ls
|
|
|
|
- name: test re-running of external script should result in same output
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool | tee output1.txt
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool | tee output2.txt
|
|
if cmp output1.txt output2.txt; then
|
|
echo "files have same output"
|
|
rm output1.txt
|
|
rm output2.txt
|
|
else
|
|
echo "re-run with same flags changed the output, result in failure"
|
|
rm output1.txt
|
|
rm output2.txt
|
|
exit 1
|
|
fi
|
|
|
|
- name: dry run external script create-external-cluster-resources.py
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool --dry-run
|
|
|
|
- name: test external script create-external-cluster-resources.py if users already exist with different caps
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# update client.csi-rbd-provisioner.1 csi user caps
|
|
# print client.csi-rbd-provisioner.1 user before update
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner.1
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth caps client.csi-rbd-provisioner.1 mon 'profile rbd, allow command "osd ls"' osd 'profile rbd' mgr 'allow rw'
|
|
# print client.csi-rbd-provisioner.1 user after update
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner.1
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool
|
|
# print client.csi-rbd-provisioner.1 user after running script
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner.1
|
|
|
|
- name: run external script create-external-cluster-resources.py unit tests
|
|
run: |
|
|
kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- python3 -m unittest /etc/ceph/create-external-cluster-resources-tests.py
|
|
|
|
- name: wait for the subvolumegroup to be created
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done"
|
|
|
|
- name: test subvolumegroup validation
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# pass the correct subvolumegroup and cephfs_filesystem flag name
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group group-a --cephfs-filesystem-name myfs
|
|
# pass the subvolumegroup name which doesn't exist
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group false-test-subvolume-group
|
|
|
|
- name: dry run test skip monitoring endpoint
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool --dry-run --skip-monitoring-endpoint
|
|
|
|
- name: test of rados namespace
|
|
run: |
|
|
kubectl create -f deploy/examples/radosnamespace.yaml
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- rbd namespace ls replicapool --format=json|jq .[0].name|grep -q "namespace-a"; do sleep 1 && echo 'waiting for the rados namespace to be created'; done"
|
|
kubectl delete -f deploy/examples/radosnamespace.yaml
|
|
|
|
- name: test rados namespace validation
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/radosnamespace1
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace radosnamespace1
|
|
# test the rados namespace which not exit for replicapool(false testing)
|
|
if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace false-test-namespace); then
|
|
echo "unexpectedly succeeded after passing the wrong rados namespace: $output"
|
|
exit 1
|
|
else
|
|
echo "script failed because wrong rados namespace was passed"
|
|
fi
|
|
|
|
- name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true
|
|
|
|
- name: test external script with restricted_auth_permission flag and with --cluster-name legacy flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cluster-name rookstorage --restricted-auth-permission true
|
|
|
|
- name: test external script with restricted_auth_permission flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --cephfs-filesystem-name myfs --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true
|
|
|
|
- name: test the upgrade flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# print existing client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth ls
|
|
# update the existing non-restricted client auth with the new ones
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --upgrade
|
|
# print upgraded client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth ls
|
|
|
|
- name: test the upgrade flag for restricted auth user
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# print existing client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-node-rookstorage-replicapool
|
|
# restricted auth user need to provide --rbd-data-pool-name,
|
|
# --k8s-cluster-name and --run-as-user flag while upgrading
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --upgrade --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --run-as-user client.csi-rbd-node-rookstorage-replicapool
|
|
# print upgraded client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-node-rookstorage-replicapool
|
|
|
|
- name: validate-rgw-endpoint
|
|
run: |
|
|
rgw_endpoint=$(kubectl get service -n rook-ceph -l rgw=store-a | awk '/rgw/ {print $3":80"}')
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# pass the valid rgw-endpoint of same ceph cluster
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-zone-name store-a --rgw-zonegroup-name store-a --rgw-realm-name store-a 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
|
|
tests/scripts/github-action-helper.sh check_empty_file output.txt
|
|
rm -f output.txt
|
|
# pass the invalid rgw-endpoint of different ceph cluster
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint 10.108.96.128:80 --rgw-zone-name store-a --rgw-zonegroup-name store-a --rgw-realm-name store-a 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
|
|
if [ -s output.txt ]; then
|
|
echo "script run completed with stderr error after passing the wrong rgw-endpoint: $output"
|
|
rm -f output.txt
|
|
else
|
|
echo "no stderr error even wrong endpoint was provided"
|
|
rm -f output.txt
|
|
exit 1
|
|
fi
|
|
# pass the valid rgw-endpoint of same ceph cluster with --rgw-tls-cert-path
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-tls-cert-path my-cert --rgw-zone-name store-a --rgw-zonegroup-name store-a --rgw-realm-name store-a 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
|
|
tests/scripts/github-action-helper.sh check_empty_file output.txt
|
|
rm -f output.txt
|
|
# pass the valid rgw-endpoint of same ceph cluster with --rgw-skip-tls
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-skip-tls true --rgw-zone-name store-a --rgw-zonegroup-name store-a --rgw-realm-name store-a 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
|
|
tests/scripts/github-action-helper.sh check_empty_file output.txt
|
|
rm -f output.txt
|
|
|
|
- name: validate multisite
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create realm
|
|
kubectl -n rook-ceph exec $toolbox -- radosgw-admin realm create --rgw-realm=realm1
|
|
# pass correct realm
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm1
|
|
# pass wrong realm
|
|
if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm3); then
|
|
echo "script run completed with stderr error after passing the wrong realm: $output"
|
|
else
|
|
echo "script failed because wrong realm was passed"
|
|
fi
|
|
|
|
- name: test topology flags
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create 3 replica-1 pools
|
|
sed -i 's/replicapool/replica1a/' deploy/examples/pool-test.yaml
|
|
kubectl create -f deploy/examples/pool-test.yaml
|
|
sed -i 's/replica1a/replica1b/' deploy/examples/pool-test.yaml
|
|
kubectl create -f deploy/examples/pool-test.yaml
|
|
sed -i 's/replica1b/replica1c/' deploy/examples/pool-test.yaml
|
|
kubectl create -f deploy/examples/pool-test.yaml
|
|
# bring back the original file
|
|
sed -i 's/replica1c/replicapool/' deploy/examples/pool-test.yaml
|
|
|
|
# check and wait for the pools to get ready
|
|
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1a -nrook-ceph
|
|
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1b -nrook-ceph
|
|
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1c -nrook-ceph
|
|
|
|
# pass correct flags
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03
|
|
# pass the pool which is not exists
|
|
if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then
|
|
echo "script run completed with stderr error after passing the wrong pools: $output"
|
|
else
|
|
echo "script failed because wrong pools doesn't exist"
|
|
fi
|
|
# dont pass all topology flags
|
|
if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then
|
|
echo "script run completed with stderr error after passing the wrong flags: $output"
|
|
else
|
|
echo "script failed because topology-failure-domain-label is missing"
|
|
fi
|
|
|
|
- name: test enable v2 mon port
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --v2-port-enable
|
|
|
|
- name: test config file flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create `radosNamespace2` rados-namespace for `replicapool` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/radosnamespace2
|
|
|
|
# validate `--rados-namespace`, config file value is radosnamespace2, cli arg flag is set to radosnamespace1 and no default is set
|
|
# command line arg should be used
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace radosnamespace1 --config-file /etc/ceph/external-config.ini
|
|
|
|
# validate the `rbd-data-pool-name`,config value set to replicapoolconfig, command line arg is not set and no default
|
|
# config value should be used
|
|
sed -i 's/replicapool/replicapoolconfig/' deploy/examples/pool-test.yaml
|
|
kubectl create -f deploy/examples/pool-test.yaml
|
|
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replicapoolconfig -nrook-ceph
|
|
# create `radosNamespace2` rados-namespace for `replicapoolconfig` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapoolconfig/radosnamespace2
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --config-file /etc/ceph/external-config.ini
|
|
|
|
# validate the default `format`` flag, config file value is not set and no cmd line argument is set and default value is `Json``,
|
|
# default value should be used
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --config-file /etc/ceph/external-config.ini
|
|
|
|
- name: test key rotation with --cephx-key-rotate
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
output1=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs)
|
|
|
|
# check if it create a csi rbd, cephfs Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node.1 client.csi-rbd-provisioner.1 client.csi-cephfs-node.1 client.csi-cephfs-provisioner.1 client.healthchecker.1
|
|
|
|
# rotate the key
|
|
output2=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs --cephx-key-rotate rotate)
|
|
|
|
# check if it create a csi rbd, cephfs and Healthchecker rotated keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node.2 client.csi-rbd-provisioner.2 client.csi-cephfs-node.2 client.csi-cephfs-provisioner.2 client.healthchecker.2
|
|
|
|
# check the content of the secret, it should be changed
|
|
if [ "$output1" != "$output2" ]; then
|
|
echo "csi rbd and cephfs key rotated successfully"
|
|
else
|
|
echo "csi rbd and cephfs key not rotated"
|
|
exit 1
|
|
fi
|
|
|
|
# revert the key
|
|
output3=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs --cephx-key-rotate revert)
|
|
# check if it revert the key, it should be same as output1
|
|
if [ "$output1" == "$output3" ]; then
|
|
echo "csi rbd and cephfs key reverted successfully"
|
|
else
|
|
echo "csi rbd and cephfs key not reverted"
|
|
exit 1
|
|
fi
|
|
|
|
# if --cephx-key-rotate is not set provide the latest key
|
|
output4=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs)
|
|
if [ "$output2" == "$output4" ]; then
|
|
echo "csi rbd and cephfs key uses latest generation"
|
|
else
|
|
echo "csi rbd and cephfs key doesn't uses latest generation"
|
|
exit 1
|
|
fi
|
|
|
|
- name: test key rotation with --cephx-key-rotate in multi tenant case, where --restricted-auth-permission is set
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create tenant1 and verify
|
|
# create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/rn1
|
|
output1=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true)
|
|
|
|
# check if it create a csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker
|
|
|
|
# rotate tenant1 and verify
|
|
output2=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate)
|
|
# check if it create rotated csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1.1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1.1 client.csi-cephfs-provisioner-rookstorage-myfs.1 client.csi-cephfs-node-rookstorage-myfs.1 client.healthchecker.1
|
|
|
|
# revert tenant1 and verify
|
|
output3=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate revert)
|
|
# check if it create a revert csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker
|
|
|
|
# check if it revert the key, it should be same as output1
|
|
if [ "$output1" == "$output3" ]; then
|
|
echo "csi rbd and cephfs key reverted successfully"
|
|
else
|
|
echo "csi rbd and cephfs key not reverted"
|
|
exit 1
|
|
fi
|
|
|
|
# create tenant2 and verify
|
|
# create `radosNamespace2` rados-namespace for `replicapool` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/rn2
|
|
output4=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true)
|
|
|
|
# check if it create a csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker
|
|
|
|
# rotate tenant2 and verify
|
|
output5=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate)
|
|
# check if it create rotated csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2.1 client.csi-rbd-provisioner-rookstorage-replicapool-rn2.1 client.csi-cephfs-provisioner-rookstorage-myfs.1 client.csi-cephfs-node-rookstorage-myfs.1 client.healthchecker.1
|
|
|
|
# rotate tenant1 (again) and verify
|
|
output6=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate)
|
|
# check if it create rotated csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1.2 client.csi-rbd-provisioner-rookstorage-replicapool-rn1.2 client.csi-cephfs-provisioner-rookstorage-myfs.2 client.csi-cephfs-node-rookstorage-myfs.2 client.healthchecker.2
|
|
|
|
# verify if tenant1 is rotated, tenant2 keys should not be rotated
|
|
# only rbd keys have separation of tenant1 and tenant2, and cephfs keys are shared
|
|
tests/scripts/github-action-helper.sh check_keys_not_exists client.csi-rbd-node-rookstorage-replicapool-rn2.2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2.2
|
|
|
|
# revert tenant2 and verify
|
|
output7=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate revert)
|
|
# check if it create a reverted csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker
|
|
|
|
# check if it revert the key, it should be same as output1
|
|
if [ "$output4" == "$output7" ]; then
|
|
echo "csi rbd and cephfs key reverted successfully"
|
|
else
|
|
echo "csi rbd and cephfs key not reverted"
|
|
exit 1
|
|
fi
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: validate csi-addons
|
|
run: |
|
|
tests/scripts/csiaddons.sh verify_crd_created
|
|
tests/scripts/csiaddons.sh verify_container_is_running
|
|
|
|
- name: test osd removal jobs
|
|
run: |
|
|
kubectl -n rook-ceph delete deploy/rook-ceph-operator
|
|
kubectl -n rook-ceph delete deploy/rook-ceph-osd-1 --grace-period=0 --force
|
|
sed -i 's/<OSD-IDs>/1/' deploy/examples/osd-purge.yaml
|
|
# the CI must force the deletion since we use replica 1 on 2 OSDs
|
|
sed -i 's/false/true/' deploy/examples/osd-purge.yaml
|
|
sed -i 's|rook/ceph:.*|rook/ceph:local-build|' deploy/examples/osd-purge.yaml
|
|
kubectl -n rook-ceph create -f deploy/examples/osd-purge.yaml
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- ceph status
|
|
# wait until osd.1 is removed from the osd tree
|
|
timeout 120 sh -c "while kubectl -n rook-ceph exec $toolbox -- ceph osd tree|grep -qE 'osd.1'; do echo 'waiting for ceph osd 1 to be purged'; sleep 1; done"
|
|
kubectl -n rook-ceph exec $toolbox -- ceph status
|
|
kubectl -n rook-ceph exec $toolbox -- ceph osd tree
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
raw-disk-with-object:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: prepare loop devices for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh prepare_loop_devices 1
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
export ALLOW_LOOP_DEVICES=true
|
|
tests/scripts/github-action-helper.sh deploy_cluster loop
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
tests/scripts/github-action-helper.sh create_operator_toolbox
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
|
|
- name: wait for object stores to be ready
|
|
run: |
|
|
tests/scripts/validate_cluster.sh rgw store-a
|
|
tests/scripts/validate_cluster.sh rgw store-b
|
|
|
|
- name: test toolbox-operator-image pod
|
|
run: |
|
|
# waiting for toolbox operator image pod to get ready
|
|
kubectl -n rook-ceph wait --for=condition=ready pod -l app=rook-ceph-tools-operator-image --timeout=180s
|
|
|
|
- name: check s5cmd version in the toolbox image
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools-operator-image -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
s5cmd_version="$(kubectl -n rook-ceph exec ${toolbox} -- /usr/local/bin/s5cmd version)"
|
|
echo ${s5cmd_version} | grep -q "^v2.3.0" || {
|
|
echo " Error: the version of s5cmd version in the toolbox is not the expected v2.3.0 but ${s5cmd_version}"
|
|
exit 1
|
|
}
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
two-osds-in-device:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster two_osds_in_device
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
osd-with-metadata-partition-device:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD metadata partition
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --bluestore-type block.db --osd-count 1
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_partition_device
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
osd-with-metadata-device:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: create LV on disk
|
|
run: |
|
|
dd if=/dev/zero of=test-rook.img bs=1 count=0 seek=10G
|
|
# If we use metadata device, both data devices and metadata devices should be logical volumes or raw devices
|
|
tests/scripts/github-action-helper.sh create_LV_on_disk $(sudo losetup --find --show test-rook.img)
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_device
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster encryption
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster encryption
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
lvm:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: create LV on disk
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster lvm
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
pvc:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: prepare loop devices for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh prepare_loop_devices 1
|
|
|
|
- name: create cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/loopDevicePV.sh 1
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
yq write -i deploy/examples/operator.yaml "data.ROOK_CEPH_ALLOW_LOOP_DEVICES" --style=double "true"
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 3
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 3
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 3
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
lsblk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo head --bytes=60 /dev/loop1
|
|
|
|
- name: recreate cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/loopDevicePV.sh 1
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: redeploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 3
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 3
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
lsblk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo head --bytes=60 /dev/loop1
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
pvc-db:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create bluestore partitions and PVCs
|
|
run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
|
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
pvc-db-wal:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create bluestore partitions and PVCs for wal
|
|
run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy rook
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
|
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
kubectl -n rook-ceph get pods
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo lsblk
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: create cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
kubectl -n rook-ceph get secrets
|
|
sudo lsblk
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo lsblk
|
|
|
|
- name: recreate cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: redeploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
kubectl -n rook-ceph get secrets
|
|
sudo lsblk
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo lsblk
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-db:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create bluestore partitions and PVCs
|
|
run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
kubectl -n rook-ceph get pods
|
|
kubectl -n rook-ceph get secrets
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-db-wal:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create bluestore partitions and PVCs for wal
|
|
run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy rook
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}'
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
kubectl -n rook-ceph get pods
|
|
kubectl -n rook-ceph get secrets
|
|
|
|
- name: wait and verify key rotation
|
|
run: tests/scripts/github-action-helper.sh verify_key_rotation
|
|
|
|
- name: test osd deployment removal and re-hydration
|
|
run: |
|
|
kubectl -n rook-ceph delete deploy/rook-ceph-osd-0
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-kms-vault-token-auth:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: create cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy vault
|
|
run: tests/scripts/deploy-validate-vault.sh deploy
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
|
|
yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2
|
|
kubectl create -f tests/manifests/test-object.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
tests/scripts/validate_cluster.sh rgw my-store
|
|
kubectl -n rook-ceph get pods
|
|
kubectl -n rook-ceph get secrets
|
|
|
|
- name: validate osd vault
|
|
run: |
|
|
tests/scripts/deploy-validate-vault.sh validate_osd
|
|
sudo lsblk
|
|
|
|
- name: wait and verify key rotation
|
|
run: tests/scripts/deploy-validate-vault.sh validate_key_rotation rook/ver1
|
|
|
|
- name: validate rgw vault kv
|
|
run: |
|
|
tests/scripts/deploy-validate-vault.sh validate_rgw
|
|
|
|
- name: validate rgw vault transit
|
|
run: |
|
|
kubectl delete -f tests/manifests/test-object.yaml
|
|
yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_SECRET_ENGINE" transit
|
|
timeout 120 bash -c 'while kubectl -n rook-ceph get cephobjectstore my-store; do echo "waiting for objectstore my-store to delete"; sleep 5; done'
|
|
echo "wait for rgw pod to be deleted"
|
|
kubectl wait --for=delete pod -l app=rook-ceph-rgw -n rook-ceph --timeout=100s
|
|
kubectl create -f tests/manifests/test-object.yaml
|
|
tests/scripts/validate_cluster.sh rgw my-store
|
|
tests/scripts/deploy-validate-vault.sh validate_rgw
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-kms-vault-k8s-auth:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: create cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy vault
|
|
run: KUBERNETES_AUTH=true tests/scripts/deploy-validate-vault.sh deploy
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-k8s-auth.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
kubectl -n rook-ceph get pods
|
|
kubectl -n rook-ceph get secrets
|
|
|
|
- name: validate osd vault
|
|
run: |
|
|
tests/scripts/deploy-validate-vault.sh validate_osd
|
|
sudo lsblk
|
|
|
|
- name: wait and verify key rotation
|
|
run: KUBERNETES_AUTH=true tests/scripts/deploy-validate-vault.sh validate_key_rotation rook/ver1
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
lvm-pvc:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create LV on disk
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK
|
|
tests/scripts/localPathPV.sh /dev/test-rook-vg/test-rook-lv
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
multi-cluster-mirroring:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk into two partitions
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2
|
|
sudo lsblk
|
|
|
|
- name: deploy first cluster rook
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_first_rook_cluster
|
|
cd deploy/examples/
|
|
sed -i "/resources:/,/ # priorityClassName:/d" rbdmirror.yaml
|
|
sed -i "/resources:/,/ # priorityClassName:/d" filesystem-mirror.yaml
|
|
kubectl create -f rbdmirror.yaml -f filesystem-mirror.yaml
|
|
|
|
# cephfs-mirroring is a push operation
|
|
# running bootstrap create on secondary and bootstrap import on primary. mirror daemons on primary.
|
|
- name: deploy second cluster rook
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_second_rook_cluster
|
|
cd deploy/examples/
|
|
sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' rbdmirror.yaml
|
|
kubectl create -f rbdmirror.yaml
|
|
|
|
- name: wait for ceph cluster 1 to be ready
|
|
run: |
|
|
mkdir -p test
|
|
tests/scripts/validate_cluster.sh osd 1
|
|
kubectl -n rook-ceph get pods
|
|
|
|
- name: create replicated mirrored pool on cluster 1
|
|
run: |
|
|
cd deploy/examples/
|
|
yq w -i pool-test.yaml spec.mirroring.enabled true
|
|
yq w -i pool-test.yaml spec.mirroring.mode image
|
|
kubectl create -f pool-test.yaml
|
|
timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to be created on cluster 1" && sleep 1; done'
|
|
|
|
- name: create replicated mirrored pool 2 on cluster 1
|
|
run: |
|
|
cd deploy/examples/
|
|
yq w -i pool-test.yaml metadata.name replicapool2
|
|
kubectl create -f pool-test.yaml
|
|
timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool2 -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to be created on cluster 1" && sleep 1; done'
|
|
yq w -i pool-test.yaml metadata.name replicapool
|
|
|
|
- name: create replicated mirrored pool on cluster 2
|
|
run: |
|
|
cd deploy/examples/
|
|
yq w -i pool-test.yaml metadata.namespace rook-ceph-secondary
|
|
kubectl create -f pool-test.yaml
|
|
timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to be created on cluster 2" && sleep 1; done'
|
|
|
|
- name: create replicated mirrored pool 2 on cluster 2
|
|
run: |
|
|
cd deploy/examples/
|
|
yq w -i pool-test.yaml metadata.name replicapool2
|
|
kubectl create -f pool-test.yaml
|
|
timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to be created on cluster 2" && sleep 1; done'
|
|
|
|
- name: create images in the pools
|
|
run: |
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -ti -- rbd -p replicapool create test -s 1G
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd mirror image enable replicapool/test snapshot
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -ti -- rbd -p replicapool2 create test -s 1G
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd mirror image enable replicapool2/test snapshot
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test
|
|
|
|
- name: copy block mirror peer secret into the other cluster for replicapool
|
|
run: |
|
|
kubectl -n rook-ceph get secret pool-peer-token-replicapool -o yaml > pool-peer-token-replicapool.yaml
|
|
yq delete --inplace pool-peer-token-replicapool.yaml metadata.ownerReferences
|
|
yq write --inplace pool-peer-token-replicapool.yaml metadata.namespace rook-ceph-secondary
|
|
yq write --inplace pool-peer-token-replicapool.yaml metadata.name pool-peer-token-replicapool-config
|
|
kubectl create --namespace=rook-ceph-secondary -f pool-peer-token-replicapool.yaml
|
|
|
|
- name: copy block mirror peer secret into the other cluster for replicapool2 (using cluster global peer)
|
|
run: |
|
|
kubectl -n rook-ceph get secret cluster-peer-token-my-cluster -o yaml > cluster-peer-token-my-cluster.yaml
|
|
yq delete --inplace cluster-peer-token-my-cluster.yaml metadata.ownerReferences
|
|
yq write --inplace cluster-peer-token-my-cluster.yaml metadata.namespace rook-ceph-secondary
|
|
yq write --inplace cluster-peer-token-my-cluster.yaml metadata.name cluster-peer-token-my-cluster-config
|
|
kubectl create --namespace=rook-ceph-secondary -f cluster-peer-token-my-cluster.yaml
|
|
|
|
- name: add block mirror peer secret to the other cluster for replicapool
|
|
run: |
|
|
kubectl -n rook-ceph-secondary patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}'
|
|
|
|
- name: add block mirror peer secret to the other cluster for replicapool2 (using cluster global peer)
|
|
run: |
|
|
kubectl -n rook-ceph-secondary patch cephblockpool replicapool2 --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["cluster-peer-token-my-cluster-config"]}}}}'
|
|
|
|
- name: verify image has been mirrored for replicapool
|
|
run: |
|
|
# let's wait a bit for the image to be present
|
|
timeout 120 sh -c 'until [ "$(kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- rbd -p replicapool ls|grep -c test)" -eq 1 ]; do echo "waiting for image to be mirrored in pool replicapool" && sleep 1; done'
|
|
|
|
- name: verify image has been mirrored for replicapool2
|
|
run: |
|
|
# let's wait a bit for the image to be present
|
|
timeout 120 sh -c 'until [ "$(kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- rbd -p replicapool2 ls|grep -c test)" -eq 1 ]; do echo "waiting for image to be mirrored in pool replicapool2" && sleep 1; done'
|
|
|
|
- name: display cephblockpool and image status
|
|
run: |
|
|
timeout 80 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.mirroringStatus.summary.daemon_health}'|grep -c OK)" -eq 1 ]; do echo "waiting for mirroring status to be updated in replicapool" && sleep 1; done'
|
|
timeout 80 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool2 -o jsonpath='{.status.mirroringStatus.summary.daemon_health}'|grep -c OK)" -eq 1 ]; do echo "waiting for mirroring status to be updated in replicapool2" && sleep 1; done'
|
|
kubectl -n rook-ceph-secondary get cephblockpool replicapool -o yaml
|
|
kubectl -n rook-ceph-secondary get cephblockpool replicapool2 -o yaml
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test
|
|
|
|
- name: copy block mirror peer secret into the primary cluster for replicapool
|
|
run: |
|
|
kubectl -n rook-ceph-secondary get secret pool-peer-token-replicapool -o yaml |\
|
|
sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: pool-peer-token-replicapool/name: pool-peer-token-replicapool-config/g' |\
|
|
kubectl create --namespace=rook-ceph -f -
|
|
|
|
- name: add block mirror peer secret to the primary cluster for replicapool
|
|
run: |
|
|
kubectl -n rook-ceph patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}'
|
|
|
|
- name: wait for rook-ceph-csi-mapping-config to be updated with cluster ID
|
|
run: |
|
|
timeout 60 sh -c 'until [ "$(kubectl get cm -n rook-ceph rook-ceph-csi-mapping-config -o jsonpath='{.data.csi-mapping-config-json}' | grep -c "rook-ceph-secondary")" -eq 1 ]; do echo "waiting for rook-ceph-csi-mapping-config to be created with cluster ID mappings" && sleep 1; done'
|
|
|
|
- name: create replicated mirrored filesystem on cluster 1
|
|
run: |
|
|
PRIMARY_YAML=deploy/examples/filesystem-test-primary.yaml
|
|
cp deploy/examples/filesystem-test.yaml "$PRIMARY_YAML"
|
|
yq merge --inplace --arrays append "$PRIMARY_YAML" tests/manifests/test-fs-mirror-spec.yaml
|
|
kubectl create -f "$PRIMARY_YAML"
|
|
|
|
- name: create replicated mirrored filesystem on cluster 2
|
|
run: |
|
|
cd deploy/examples/
|
|
sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' filesystem-test.yaml
|
|
yq w -i filesystem-test.yaml spec.mirroring.enabled true
|
|
kubectl create -f filesystem-test.yaml
|
|
|
|
- name: wait for filesystem on cluster 1
|
|
run: |
|
|
timeout 300 sh -c 'until [ "$(kubectl -n rook-ceph get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done'
|
|
|
|
- name: wait for filesystem on cluster 2
|
|
run: |
|
|
timeout 300 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done'
|
|
|
|
- name: copy filesystem mirror peer secret from the secondary cluster to the primary one
|
|
run: |
|
|
kubectl -n rook-ceph-secondary get secret fs-peer-token-myfs -o yaml |\
|
|
sed '/ownerReferences/,+6d' |\
|
|
sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: fs-peer-token-myfs/name: fs-peer-token-myfs-config/g' |\
|
|
kubectl create --namespace=rook-ceph -f -
|
|
|
|
- name: add filesystem mirror peer secret to the primary cluster
|
|
run: |
|
|
kubectl -n rook-ceph patch cephfilesystem myfs --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["fs-peer-token-myfs-config"]}}}}'
|
|
|
|
- name: verify fs mirroring is working
|
|
run: |
|
|
timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -gt 3 ]; do echo "waiting for connection to peer" && sleep 1; done'
|
|
sockets=$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/)
|
|
status=$(for socket in $sockets; do minikube kubectl -- -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon /var/run/ceph/$socket help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done)
|
|
if [ "${#status}" -lt 1 ]; then echo "peer addition failed" && exit 1; fi
|
|
|
|
- name: display cephfilesystem and fs mirror daemon status
|
|
run: |
|
|
kubectl -n rook-ceph get cephfilesystem myfs -o yaml
|
|
# the check is not super ideal since 'mirroring_failed' is only displayed when there is a failure but not when it's working...
|
|
timeout 60 sh -c 'while [ "$(kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror daemon status|jq -r '.[0].filesystems[0]'|grep -c "mirroring_failed")" -eq 1 ]; do echo "waiting for filesystem to be mirrored" && sleep 1; done'
|
|
|
|
- name: Create subvolume on primary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume create myfs testsubvolume
|
|
|
|
- name: Create subvolume of same name on secondary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- ceph fs subvolume create myfs testsubvolume
|
|
|
|
- name: Deploy Direct Tools pod on primary cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/direct-mount.yaml
|
|
|
|
- name: Deploy Direct Tools pod on secondary cluster
|
|
run: |
|
|
sed -i "s/rook-ceph # namespace/rook-ceph-secondary # namespace/" deploy/examples/direct-mount.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/direct-mount.yaml
|
|
|
|
- name: Configure a directory for snapshot mirroring on primary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror enable myfs
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror add myfs /volumes/_nogroup/testsubvolume/
|
|
|
|
- name: make sure that snapshot mirror is enabled on the secondary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- ceph fs snapshot mirror enable myfs
|
|
|
|
- name: Create 3 snapshots on cluster primary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume snapshot create myfs testsubvolume snap1
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume snapshot create myfs testsubvolume snap2
|
|
|
|
- name: Get the peer and verify the peer synchronization status that snaps have synced on secondary cluster
|
|
run: |
|
|
exec_fs_mirror='kubectl -n rook-ceph exec deploy/rook-ceph-fs-mirror --'
|
|
mirror_daemon=$($exec_fs_mirror ls /var/run/ceph/ | grep "fs-mirror" | head -n 1)
|
|
# timeout 45 bash -x <<EOF
|
|
# while
|
|
# clusterfsid=\$($exec_fs_mirror ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror status myfs@1 |jq -r '.peers|keys[]')
|
|
# [ -z "\$clusterfsid" ]
|
|
# do echo "Waiting for the clusterfsid to get populated." && sleep 1
|
|
# done
|
|
# EOF
|
|
# clusterfsid=$($exec_fs_mirror ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror status myfs@1 |jq -r '.peers|keys[]')
|
|
# echo $clusterfsid
|
|
# kubectl -n rook-ceph-secondary wait pod -l app=rook-direct-mount --for condition=Ready --timeout=400s
|
|
# kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- mkdir /tmp/registry
|
|
# mon_endpoints=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- grep mon_host /etc/ceph/ceph.conf | awk '{print $3}')
|
|
# my_secret=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- grep key /etc/ceph/keyring | awk '{print $3}')
|
|
# kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- mount -t ceph -o mds_namespace=myfs,name=admin,secret=$my_secret $mon_endpoints:/ /tmp/registry
|
|
# num_snaps_target=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- ls -lhsa /tmp/registry/volumes/_nogroup/testsubvolume/.snap|grep snap|wc -l)
|
|
# snaps=$(kubectl -n rook-ceph exec deploy/rook-ceph-fs-mirror -- ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror peer status myfs@1 $clusterfsid|jq -r '."/volumes/_nogroup/testsubvolume"."snaps_synced"')
|
|
# echo "snapshots: $snaps"
|
|
# if [ $num_snaps_target = $snaps ]
|
|
# then echo "Snapshots have synced."
|
|
# else echo "Snapshots have not synced. NEEDS INVESTIGATION"
|
|
# fi
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
additional-namespace: rook-ceph-secondary
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
rgw-multisite-testing:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: install additional deps for object testing
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
sudo apt-get install -y s3cmd
|
|
|
|
- name: use local disk into two partitions
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2
|
|
sudo lsblk
|
|
|
|
- name: deploy first cluster rook
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_first_rook_cluster
|
|
kubectl create -f deploy/examples/object-multisite-test.yaml
|
|
# wait for multisite-store to be created
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectstore multisite-store rook-ceph 600
|
|
|
|
- name: prep second cluster pull realm config
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
cd deploy/examples/
|
|
IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
|
|
yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80
|
|
BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
|
|
BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}")
|
|
sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml
|
|
sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml
|
|
|
|
- name: deploy second cluster rook
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_second_rook_cluster
|
|
kubectl create -f deploy/examples/object-multisite-pull-realm-test.yaml
|
|
# wait for realms to be pulled and zone-b-multisite-store to be created
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectstore zone-b-multisite-store rook-ceph-secondary 600
|
|
|
|
- name: wait for both ceph clusters to be ready
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph
|
|
tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph-secondary
|
|
|
|
- name: write an object to one cluster, read from the other
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
tests/scripts/github-action-helper.sh test_multisite_object_replication
|
|
|
|
# if this test fails, it could mean the RGW `period get` or `period update` output has changed
|
|
- name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
ns_name_primary_object_store='"rook-ceph/multisite-store"' # double quotes intended
|
|
ns_name_secondary_object_store='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended
|
|
committed_msg="committing changes to RGW configuration period for CephObjectStore"
|
|
tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary_object_store}"
|
|
tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary_object_store}"
|
|
ns_name_primary_object_zone='"rook-ceph/zone-a"' # double quotes intended
|
|
ns_name_secondary_object_zone='"rook-ceph-secondary/zone-b"' # double quotes intended
|
|
tests/scripts/github-action-helper.sh restart_operator
|
|
not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore"
|
|
tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary_object_zone}" 600
|
|
tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary_object_zone}" 600
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
additional-namespace: rook-ceph-secondary
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
nvmeof-protocol:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ["quay.io/ceph/ceph:v20"]
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: install nvme initiator prerequisites
|
|
run: tests/scripts/github-action-helper.sh install_nvme_initiator_prerequisites
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: deploy cluster without ceph-csi operator
|
|
run: |
|
|
OPERATOR_MANIFEST="deploy/examples/operator.yaml"
|
|
sed -i 's|# ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.16.1"|ROOK_CSI_CEPH_IMAGE: "quay.io/nixpanic/cephcsi:nvmeof"|' "${OPERATOR_MANIFEST}"
|
|
sed -i 's|ROOK_USE_CSI_OPERATOR: "true"|ROOK_USE_CSI_OPERATOR: "false"|' "${OPERATOR_MANIFEST}"
|
|
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "${OPERATOR_MANIFEST}"
|
|
|
|
export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
yq w -i -d0 deploy/examples/cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"
|
|
kubectl create -f deploy/examples/cluster-test.yaml
|
|
tests/scripts/github-action-helper.sh deploy_toolbox
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
env:
|
|
CSI_POD_MIN_COUNT: "6"
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
|
|
- name: test ceph-csi-nvmeof plugin restart and data persistence
|
|
run: tests/scripts/github-action-helper.sh test_csi_nvmeof_workload
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-kms-ibm-kp:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: run encryption KMS IBM Key Protect
|
|
uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
|
|
if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
|
|
env:
|
|
IBM_KP_SERVICE_INSTANCE_ID: ${{ secrets.IBM_INSTANCE_ID }}
|
|
IBM_KP_SERVICE_API_KEY: ${{ secrets.IBM_SERVICE_API_KEY }}
|
|
with:
|
|
ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }}
|
|
ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
artifact-name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
ceph-image: ${{ matrix.ceph-image }}
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
multus-public-and-cluster:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-multus-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: Setup multus
|
|
run: ./tests/scripts/multus/setup-multus.sh
|
|
|
|
- name: Set up multus prerequisite host routing
|
|
run: kubectl create -f tests/scripts/multus/host-cfg-ds.yaml
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: Install public and cluster NADs in rook-ceph namespace
|
|
run: kubectl create -f tests/scripts/multus/rook-ceph-public-cluster-nads.yaml
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
|
|
|
|
export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
yq write -i deploy/examples/cluster-multus-test.yaml "spec.storage.deviceFilter" "${BLOCK}"
|
|
kubectl create -f deploy/examples/cluster-multus-test.yaml
|
|
|
|
tests/scripts/github-action-helper.sh deploy_toolbox
|
|
|
|
kubectl create -f deploy/examples/filesystem-test.yaml
|
|
kubectl create -f deploy/examples/nfs-test.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
|
|
- name: wait for cephnfs to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready nfs 1
|
|
|
|
- name: check multus connections
|
|
run: tests/scripts/github-action-helper.sh test_multus_connections
|
|
|
|
- name: test ceph-csi-rbd plugin restart
|
|
run: tests/scripts/github-action-helper.sh test_csi_rbd_workload
|
|
|
|
- name: test ceph-csi-cephfs plugin restart
|
|
run: tests/scripts/github-action-helper.sh test_csi_cephfs_workload
|
|
|
|
- name: test ceph-csi-nfs plugin restart
|
|
run: tests/scripts/github-action-helper.sh test_csi_nfs_workload
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
additional-namespace: kube-system
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
two-object-one-zone:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster
|
|
tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph 240
|
|
|
|
- name: create CephBlockPool(s), CephObjectZone, and CephObjectStore(s)
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
kubectl create -f deploy/examples/object-separate-pools-test.yaml
|
|
kubectl create -f deploy/examples/two-object-one-zone-test.yaml
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectzone object-separate-pools rook-ceph 600
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectstore two-object-one-zone-alpha rook-ceph 600
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectstore two-object-one-zone-beta rook-ceph 600
|
|
|
|
- name: check for pools created by RGW that are unexpected
|
|
run: tests/scripts/github-action-helper.sh test_object_separate_pools
|
|
|
|
- name: verify that only one zone exists
|
|
run: |
|
|
[[ $(tests/scripts/github-action-helper.sh radosgw-admin zone list | jq '.zones | length') == "1" ]]
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|