Niels de Vos
31fdb73eae
Signed-off-by: Niels de Vos <ndevos@ibm.com>
(cherry picked from commit 9262275a6e)
2224 lines
113 KiB
YAML
2224 lines
113 KiB
YAML
name: Reusable canary integration tests
|
|
on:
|
|
# ONLY on.workflow_call ; call this from other files if needed
|
|
workflow_call:
|
|
inputs:
|
|
ceph_images:
|
|
description: "JSON list of Ceph images for creating Ceph cluster"
|
|
default: '["quay.io/ceph/ceph:v19"]'
|
|
type: string
|
|
kubernetes-version:
|
|
description: kubernetes version to use for the workflow
|
|
default: '["v1.35.0"]'
|
|
type: string
|
|
|
|
defaults:
|
|
run:
|
|
# reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
canary:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: setup csi-addons
|
|
run: tests/scripts/csiaddons.sh setup_csiaddons
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready all 2
|
|
|
|
- name: wait for ceph mgr to be ready
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr|grep -Eosq \"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\" ; do sleep 1 && echo 'waiting for the manager IP to be available'; done"
|
|
mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr)
|
|
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done"
|
|
|
|
- name: test osd.0 auth recovery from keyring file
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
osd_id=0
|
|
osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
if [ $osd_pod ]; then
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth del osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be deleted'; done";
|
|
kubectl -n rook-ceph delete pod $osd_pod;
|
|
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth get osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be recovered'; done";
|
|
osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}');
|
|
kubectl -n rook-ceph wait --for=condition=Ready pod/$osd_pod --timeout=120s;
|
|
else
|
|
echo "osd $osd_id not found, skipping test";
|
|
fi
|
|
|
|
- name: test external script create-external-cluster-resources.py
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- mkdir -p /etc/ceph/test-data
|
|
kubectl -n rook-ceph cp tests/external-cluster/ceph-status-out $toolbox:/etc/ceph/test-data/
|
|
kubectl -n rook-ceph cp tests/external-cluster/external-config.ini $toolbox:/etc/ceph/
|
|
kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources.py $toolbox:/etc/ceph
|
|
kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources-tests.py $toolbox:/etc/ceph
|
|
timeout 10 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool; do echo 'waiting for script to succeed' && sleep 1; done"
|
|
# print existing client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth ls
|
|
|
|
- name: test re-running of external script should result in same output
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool | tee output1.txt
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool | tee output2.txt
|
|
if cmp output1.txt output2.txt; then
|
|
echo "files have same output"
|
|
rm output1.txt
|
|
rm output2.txt
|
|
else
|
|
echo "re-run with same flags changed the output, result in failure"
|
|
rm output1.txt
|
|
rm output2.txt
|
|
exit 1
|
|
fi
|
|
|
|
- name: dry run external script create-external-cluster-resources.py
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool --dry-run
|
|
|
|
- name: test external script create-external-cluster-resources.py if users already exist with different caps
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# update client.csi-rbd-provisioner.1 csi user caps
|
|
# print client.csi-rbd-provisioner.1 user before update
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner.1
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth caps client.csi-rbd-provisioner.1 mon 'profile rbd, allow command "osd ls"' osd 'profile rbd' mgr 'allow rw'
|
|
# print client.csi-rbd-provisioner.1 user after update
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner.1
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool
|
|
# print client.csi-rbd-provisioner.1 user after running script
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner.1
|
|
|
|
- name: run external script create-external-cluster-resources.py unit tests
|
|
run: |
|
|
kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- python3 -m unittest /etc/ceph/create-external-cluster-resources-tests.py
|
|
|
|
- name: wait for the subvolumegroup to be created
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done"
|
|
|
|
- name: test subvolumegroup validation
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# pass the correct subvolumegroup and cephfs_filesystem flag name
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group group-a --cephfs-filesystem-name myfs
|
|
# pass the subvolumegroup name which doesn't exist
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group false-test-subvolume-group
|
|
|
|
- name: dry run test skip monitoring endpoint
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool --dry-run --skip-monitoring-endpoint
|
|
|
|
- name: test of rados namespace
|
|
run: |
|
|
kubectl create -f deploy/examples/radosnamespace.yaml
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- rbd namespace ls replicapool --format=json|jq .[0].name|grep -q "namespace-a"; do sleep 1 && echo 'waiting for the rados namespace to be created'; done"
|
|
kubectl delete -f deploy/examples/radosnamespace.yaml
|
|
|
|
- name: test rados namespace validation
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/radosnamespace1
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace radosnamespace1
|
|
# test the rados namespace which not exit for replicapool(false testing)
|
|
if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace false-test-namespace); then
|
|
echo "unexpectedly succeeded after passing the wrong rados namespace: $output"
|
|
exit 1
|
|
else
|
|
echo "script failed because wrong rados namespace was passed"
|
|
fi
|
|
|
|
- name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true
|
|
|
|
- name: test external script with restricted_auth_permission flag and with --cluster-name legacy flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cluster-name rookstorage --restricted-auth-permission true
|
|
|
|
- name: test external script with restricted_auth_permission flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --cephfs-filesystem-name myfs --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true
|
|
|
|
- name: test the upgrade flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# print existing client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth ls
|
|
# update the existing non-restricted client auth with the new ones
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --upgrade
|
|
# print upgraded client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth ls
|
|
|
|
- name: test the upgrade flag for restricted auth user
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# print existing client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-node-rookstorage-replicapool
|
|
# restricted auth user need to provide --rbd-data-pool-name,
|
|
# --k8s-cluster-name and --run-as-user flag while upgrading
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --upgrade --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --run-as-user client.csi-rbd-node-rookstorage-replicapool
|
|
# print upgraded client auth
|
|
kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-node-rookstorage-replicapool
|
|
|
|
- name: validate-rgw-endpoint
|
|
run: |
|
|
rgw_endpoint=$(kubectl get service -n rook-ceph -l rgw=store-a | awk '/rgw/ {print $3":80"}')
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# pass the valid rgw-endpoint of same ceph cluster
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-zone-name store-a --rgw-zonegroup-name store-a --rgw-realm-name store-a 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
|
|
tests/scripts/github-action-helper.sh check_empty_file output.txt
|
|
rm -f output.txt
|
|
# pass the invalid rgw-endpoint of different ceph cluster
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint 10.108.96.128:80 --rgw-zone-name store-a --rgw-zonegroup-name store-a --rgw-realm-name store-a 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
|
|
if [ -s output.txt ]; then
|
|
echo "script run completed with stderr error after passing the wrong rgw-endpoint: $output"
|
|
rm -f output.txt
|
|
else
|
|
echo "no stderr error even wrong endpoint was provided"
|
|
rm -f output.txt
|
|
exit 1
|
|
fi
|
|
# pass the valid rgw-endpoint of same ceph cluster with --rgw-tls-cert-path
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-tls-cert-path my-cert --rgw-zone-name store-a --rgw-zonegroup-name store-a --rgw-realm-name store-a 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
|
|
tests/scripts/github-action-helper.sh check_empty_file output.txt
|
|
rm -f output.txt
|
|
# pass the valid rgw-endpoint of same ceph cluster with --rgw-skip-tls
|
|
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-skip-tls true --rgw-zone-name store-a --rgw-zonegroup-name store-a --rgw-realm-name store-a 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
|
|
tests/scripts/github-action-helper.sh check_empty_file output.txt
|
|
rm -f output.txt
|
|
|
|
- name: validate multisite
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create realm
|
|
kubectl -n rook-ceph exec $toolbox -- radosgw-admin realm create --rgw-realm=realm1
|
|
# pass correct realm
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm1
|
|
# pass wrong realm
|
|
if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm3); then
|
|
echo "script run completed with stderr error after passing the wrong realm: $output"
|
|
else
|
|
echo "script failed because wrong realm was passed"
|
|
fi
|
|
|
|
- name: test topology flags
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create 3 replica-1 pools
|
|
sed -i 's/replicapool/replica1a/' deploy/examples/pool-test.yaml
|
|
kubectl create -f deploy/examples/pool-test.yaml
|
|
sed -i 's/replica1a/replica1b/' deploy/examples/pool-test.yaml
|
|
kubectl create -f deploy/examples/pool-test.yaml
|
|
sed -i 's/replica1b/replica1c/' deploy/examples/pool-test.yaml
|
|
kubectl create -f deploy/examples/pool-test.yaml
|
|
# bring back the original file
|
|
sed -i 's/replica1c/replicapool/' deploy/examples/pool-test.yaml
|
|
|
|
# check and wait for the pools to get ready
|
|
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1a -nrook-ceph
|
|
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1b -nrook-ceph
|
|
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1c -nrook-ceph
|
|
|
|
# pass correct flags
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03
|
|
# pass the pool which is not exists
|
|
if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then
|
|
echo "script run completed with stderr error after passing the wrong pools: $output"
|
|
else
|
|
echo "script failed because wrong pools doesn't exist"
|
|
fi
|
|
# dont pass all topology flags
|
|
if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then
|
|
echo "script run completed with stderr error after passing the wrong flags: $output"
|
|
else
|
|
echo "script failed because topology-failure-domain-label is missing"
|
|
fi
|
|
|
|
- name: test enable v2 mon port
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --v2-port-enable
|
|
|
|
- name: test config file flag
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create `radosNamespace2` rados-namespace for `replicapool` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/radosnamespace2
|
|
|
|
# validate `--rados-namespace`, config file value is radosnamespace2, cli arg flag is set to radosnamespace1 and no default is set
|
|
# command line arg should be used
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace radosnamespace1 --config-file /etc/ceph/external-config.ini
|
|
|
|
# validate the `rbd-data-pool-name`,config value set to replicapoolconfig, command line arg is not set and no default
|
|
# config value should be used
|
|
sed -i 's/replicapool/replicapoolconfig/' deploy/examples/pool-test.yaml
|
|
kubectl create -f deploy/examples/pool-test.yaml
|
|
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replicapoolconfig -nrook-ceph
|
|
# create `radosNamespace2` rados-namespace for `replicapoolconfig` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapoolconfig/radosnamespace2
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --config-file /etc/ceph/external-config.ini
|
|
|
|
# validate the default `format`` flag, config file value is not set and no cmd line argument is set and default value is `Json``,
|
|
# default value should be used
|
|
kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --config-file /etc/ceph/external-config.ini
|
|
|
|
- name: test key rotation with --cephx-key-rotate
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
output1=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs)
|
|
|
|
# check if it create a csi rbd, cephfs Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node.1 client.csi-rbd-provisioner.1 client.csi-cephfs-node.1 client.csi-cephfs-provisioner.1 client.healthchecker.1
|
|
|
|
# rotate the key
|
|
output2=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs --cephx-key-rotate rotate)
|
|
|
|
# check if it create a csi rbd, cephfs and Healthchecker rotated keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node.2 client.csi-rbd-provisioner.2 client.csi-cephfs-node.2 client.csi-cephfs-provisioner.2 client.healthchecker.2
|
|
|
|
# check the content of the secret, it should be changed
|
|
if [ "$output1" != "$output2" ]; then
|
|
echo "csi rbd and cephfs key rotated successfully"
|
|
else
|
|
echo "csi rbd and cephfs key not rotated"
|
|
exit 1
|
|
fi
|
|
|
|
# revert the key
|
|
output3=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs --cephx-key-rotate revert)
|
|
# check if it revert the key, it should be same as output1
|
|
if [ "$output1" == "$output3" ]; then
|
|
echo "csi rbd and cephfs key reverted successfully"
|
|
else
|
|
echo "csi rbd and cephfs key not reverted"
|
|
exit 1
|
|
fi
|
|
|
|
# if --cephx-key-rotate is not set provide the latest key
|
|
output4=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs)
|
|
if [ "$output2" == "$output4" ]; then
|
|
echo "csi rbd and cephfs key uses latest generation"
|
|
else
|
|
echo "csi rbd and cephfs key doesn't uses latest generation"
|
|
exit 1
|
|
fi
|
|
|
|
- name: test key rotation with --cephx-key-rotate in multi tenant case, where --restricted-auth-permission is set
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
# create tenant1 and verify
|
|
# create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/rn1
|
|
output1=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true)
|
|
|
|
# check if it create a csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker
|
|
|
|
# rotate tenant1 and verify
|
|
output2=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate)
|
|
# check if it create rotated csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1.1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1.1 client.csi-cephfs-provisioner-rookstorage-myfs.1 client.csi-cephfs-node-rookstorage-myfs.1 client.healthchecker.1
|
|
|
|
# revert tenant1 and verify
|
|
output3=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate revert)
|
|
# check if it create a revert csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker
|
|
|
|
# check if it revert the key, it should be same as output1
|
|
if [ "$output1" == "$output3" ]; then
|
|
echo "csi rbd and cephfs key reverted successfully"
|
|
else
|
|
echo "csi rbd and cephfs key not reverted"
|
|
exit 1
|
|
fi
|
|
|
|
# create tenant2 and verify
|
|
# create `radosNamespace2` rados-namespace for `replicapool` rbd data-pool
|
|
kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/rn2
|
|
output4=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true)
|
|
|
|
# check if it create a csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker
|
|
|
|
# rotate tenant2 and verify
|
|
output5=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate)
|
|
# check if it create rotated csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2.1 client.csi-rbd-provisioner-rookstorage-replicapool-rn2.1 client.csi-cephfs-provisioner-rookstorage-myfs.1 client.csi-cephfs-node-rookstorage-myfs.1 client.healthchecker.1
|
|
|
|
# rotate tenant1 (again) and verify
|
|
output6=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate)
|
|
# check if it create rotated csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1.2 client.csi-rbd-provisioner-rookstorage-replicapool-rn1.2 client.csi-cephfs-provisioner-rookstorage-myfs.2 client.csi-cephfs-node-rookstorage-myfs.2 client.healthchecker.2
|
|
|
|
# verify if tenant1 is rotated, tenant2 keys should not be rotated
|
|
# only rbd keys have separation of tenant1 and tenant2, and cephfs keys are shared
|
|
tests/scripts/github-action-helper.sh check_keys_not_exists client.csi-rbd-node-rookstorage-replicapool-rn2.2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2.2
|
|
|
|
# revert tenant2 and verify
|
|
output7=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate revert)
|
|
# check if it create a reverted csi rbd, cephfs and Healthchecker keys
|
|
tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker
|
|
|
|
# check if it revert the key, it should be same as output1
|
|
if [ "$output4" == "$output7" ]; then
|
|
echo "csi rbd and cephfs key reverted successfully"
|
|
else
|
|
echo "csi rbd and cephfs key not reverted"
|
|
exit 1
|
|
fi
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: validate csi-addons
|
|
run: |
|
|
tests/scripts/csiaddons.sh verify_crd_created
|
|
tests/scripts/csiaddons.sh verify_container_is_running
|
|
|
|
- name: test osd removal jobs
|
|
run: |
|
|
kubectl -n rook-ceph delete deploy/rook-ceph-operator
|
|
kubectl -n rook-ceph delete deploy/rook-ceph-osd-1 --grace-period=0 --force
|
|
sed -i 's/<OSD-IDs>/1/' deploy/examples/osd-purge.yaml
|
|
# the CI must force the deletion since we use replica 1 on 2 OSDs
|
|
sed -i 's/false/true/' deploy/examples/osd-purge.yaml
|
|
sed -i 's|rook/ceph:.*|rook/ceph:local-build|' deploy/examples/osd-purge.yaml
|
|
kubectl -n rook-ceph create -f deploy/examples/osd-purge.yaml
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
kubectl -n rook-ceph exec $toolbox -- ceph status
|
|
# wait until osd.1 is removed from the osd tree
|
|
timeout 120 sh -c "while kubectl -n rook-ceph exec $toolbox -- ceph osd tree|grep -qE 'osd.1'; do echo 'waiting for ceph osd 1 to be purged'; sleep 1; done"
|
|
kubectl -n rook-ceph exec $toolbox -- ceph status
|
|
kubectl -n rook-ceph exec $toolbox -- ceph osd tree
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
raw-disk-with-object:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: prepare loop devices for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh prepare_loop_devices 1
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
export ALLOW_LOOP_DEVICES=true
|
|
tests/scripts/github-action-helper.sh deploy_cluster loop
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
tests/scripts/github-action-helper.sh create_operator_toolbox
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
|
|
- name: wait for object stores to be ready
|
|
run: |
|
|
tests/scripts/validate_cluster.sh rgw store-a
|
|
tests/scripts/validate_cluster.sh rgw store-b
|
|
|
|
- name: test toolbox-operator-image pod
|
|
run: |
|
|
# waiting for toolbox operator image pod to get ready
|
|
kubectl -n rook-ceph wait --for=condition=ready pod -l app=rook-ceph-tools-operator-image --timeout=180s
|
|
|
|
- name: check s5cmd version in the toolbox image
|
|
run: |
|
|
toolbox=$(kubectl get pod -l app=rook-ceph-tools-operator-image -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
|
s5cmd_version="$(kubectl -n rook-ceph exec ${toolbox} -- /usr/local/bin/s5cmd version)"
|
|
echo ${s5cmd_version} | grep -q "^v2.3.0" || {
|
|
echo " Error: the version of s5cmd version in the toolbox is not the expected v2.3.0 but ${s5cmd_version}"
|
|
exit 1
|
|
}
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
two-osds-in-device:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster two_osds_in_device
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
osd-with-metadata-partition-device:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD metadata partition
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --bluestore-type block.db --osd-count 1
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_partition_device
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
osd-with-metadata-device:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: create LV on disk
|
|
run: |
|
|
dd if=/dev/zero of=test-rook.img bs=1 count=0 seek=10G
|
|
# If we use metadata device, both data devices and metadata devices should be logical volumes or raw devices
|
|
tests/scripts/github-action-helper.sh create_LV_on_disk $(sudo losetup --find --show test-rook.img)
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_device
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster encryption
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster encryption
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
lvm:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk as OSD
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only
|
|
|
|
- name: create LV on disk
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster lvm
|
|
tests/scripts/github-action-helper.sh deploy_all_additional_resources_on_cluster
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
pvc:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: prepare loop devices for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh prepare_loop_devices 1
|
|
|
|
- name: create cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/loopDevicePV.sh 1
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
yq write -i deploy/examples/operator.yaml "data.ROOK_CEPH_ALLOW_LOOP_DEVICES" --style=double "true"
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 3
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 3
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 3
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
lsblk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo head --bytes=60 /dev/loop1
|
|
|
|
- name: recreate cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/loopDevicePV.sh 1
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: redeploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 3
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 3
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
lsblk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo head --bytes=60 /dev/loop1
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
pvc-db:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create bluestore partitions and PVCs
|
|
run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
|
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
pvc-db-wal:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create bluestore partitions and PVCs for wal
|
|
run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy rook
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
|
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
kubectl -n rook-ceph get pods
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo lsblk
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: create cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
kubectl -n rook-ceph get secrets
|
|
sudo lsblk
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo lsblk
|
|
|
|
- name: recreate cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: redeploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
kubectl -n rook-ceph get secrets
|
|
sudo lsblk
|
|
|
|
- name: teardown cluster with cleanup policy
|
|
run: |
|
|
tests/scripts/github-action-helper.sh delete_cluster
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
sudo head --bytes=60 ${BLOCK}1
|
|
sudo head --bytes=60 ${BLOCK}2
|
|
sudo lsblk
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-db:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create bluestore partitions and PVCs
|
|
run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
kubectl -n rook-ceph get pods
|
|
kubectl -n rook-ceph get secrets
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-db-wal:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create bluestore partitions and PVCs for wal
|
|
run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy rook
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}'
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
kubectl -n rook-ceph get pods
|
|
kubectl -n rook-ceph get secrets
|
|
|
|
- name: wait and verify key rotation
|
|
run: tests/scripts/github-action-helper.sh verify_key_rotation
|
|
|
|
- name: test osd deployment removal and re-hydration
|
|
run: |
|
|
kubectl -n rook-ceph delete deploy/rook-ceph-osd-0
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-kms-vault-token-auth:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: create cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy vault
|
|
run: tests/scripts/deploy-validate-vault.sh deploy
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
|
|
yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2
|
|
kubectl create -f tests/manifests/test-object.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
tests/scripts/validate_cluster.sh rgw my-store
|
|
kubectl -n rook-ceph get pods
|
|
kubectl -n rook-ceph get secrets
|
|
|
|
- name: validate osd vault
|
|
run: |
|
|
tests/scripts/deploy-validate-vault.sh validate_osd
|
|
sudo lsblk
|
|
|
|
- name: wait and verify key rotation
|
|
run: tests/scripts/deploy-validate-vault.sh validate_key_rotation rook/ver1
|
|
|
|
- name: validate rgw vault kv
|
|
run: |
|
|
tests/scripts/deploy-validate-vault.sh validate_rgw
|
|
|
|
- name: validate rgw vault transit
|
|
run: |
|
|
kubectl delete -f tests/manifests/test-object.yaml
|
|
yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_SECRET_ENGINE" transit
|
|
timeout 120 bash -c 'while kubectl -n rook-ceph get cephobjectstore my-store; do echo "waiting for objectstore my-store to delete"; sleep 5; done'
|
|
echo "wait for rgw pod to be deleted"
|
|
kubectl wait --for=delete pod -l app=rook-ceph-rgw -n rook-ceph --timeout=100s
|
|
kubectl create -f tests/manifests/test-object.yaml
|
|
tests/scripts/validate_cluster.sh rgw my-store
|
|
tests/scripts/deploy-validate-vault.sh validate_rgw
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-kms-vault-k8s-auth:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: create cluster prerequisites
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/localPathPV.sh "$BLOCK"
|
|
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: deploy vault
|
|
run: KUBERNETES_AUTH=true tests/scripts/deploy-validate-vault.sh deploy
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-k8s-auth.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
kubectl -n rook-ceph get pods
|
|
kubectl -n rook-ceph get secrets
|
|
|
|
- name: validate osd vault
|
|
run: |
|
|
tests/scripts/deploy-validate-vault.sh validate_osd
|
|
sudo lsblk
|
|
|
|
- name: wait and verify key rotation
|
|
run: KUBERNETES_AUTH=true tests/scripts/deploy-validate-vault.sh validate_key_rotation rook/ver1
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
lvm-pvc:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: use local disk
|
|
run: tests/scripts/github-action-helper.sh use_local_disk
|
|
|
|
- name: create LV on disk
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK
|
|
tests/scripts/localPathPV.sh /dev/test-rook-vg/test-rook-lv
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
|
|
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1
|
|
|
|
- name: check-ownerreferences
|
|
run: tests/scripts/github-action-helper.sh check_ownerreferences
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
multi-cluster-mirroring:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: use local disk into two partitions
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2
|
|
sudo lsblk
|
|
|
|
- name: deploy first cluster rook
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_first_rook_cluster
|
|
cd deploy/examples/
|
|
sed -i "/resources:/,/ # priorityClassName:/d" rbdmirror.yaml
|
|
sed -i "/resources:/,/ # priorityClassName:/d" filesystem-mirror.yaml
|
|
kubectl create -f rbdmirror.yaml -f filesystem-mirror.yaml
|
|
|
|
# cephfs-mirroring is a push operation
|
|
# running bootstrap create on secondary and bootstrap import on primary. mirror daemons on primary.
|
|
- name: deploy second cluster rook
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_second_rook_cluster
|
|
cd deploy/examples/
|
|
sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' rbdmirror.yaml
|
|
kubectl create -f rbdmirror.yaml
|
|
|
|
- name: wait for ceph cluster 1 to be ready
|
|
run: |
|
|
mkdir -p test
|
|
tests/scripts/validate_cluster.sh osd 1
|
|
kubectl -n rook-ceph get pods
|
|
|
|
- name: create replicated mirrored pool on cluster 1
|
|
run: |
|
|
cd deploy/examples/
|
|
yq w -i pool-test.yaml spec.mirroring.enabled true
|
|
yq w -i pool-test.yaml spec.mirroring.mode image
|
|
kubectl create -f pool-test.yaml
|
|
timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to be created on cluster 1" && sleep 1; done'
|
|
|
|
- name: create replicated mirrored pool 2 on cluster 1
|
|
run: |
|
|
cd deploy/examples/
|
|
yq w -i pool-test.yaml metadata.name replicapool2
|
|
kubectl create -f pool-test.yaml
|
|
timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool2 -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to be created on cluster 1" && sleep 1; done'
|
|
yq w -i pool-test.yaml metadata.name replicapool
|
|
|
|
- name: create replicated mirrored pool on cluster 2
|
|
run: |
|
|
cd deploy/examples/
|
|
yq w -i pool-test.yaml metadata.namespace rook-ceph-secondary
|
|
kubectl create -f pool-test.yaml
|
|
timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to be created on cluster 2" && sleep 1; done'
|
|
|
|
- name: create replicated mirrored pool 2 on cluster 2
|
|
run: |
|
|
cd deploy/examples/
|
|
yq w -i pool-test.yaml metadata.name replicapool2
|
|
kubectl create -f pool-test.yaml
|
|
timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to be created on cluster 2" && sleep 1; done'
|
|
|
|
- name: create images in the pools
|
|
run: |
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -ti -- rbd -p replicapool create test -s 1G
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd mirror image enable replicapool/test snapshot
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -ti -- rbd -p replicapool2 create test -s 1G
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd mirror image enable replicapool2/test snapshot
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test
|
|
|
|
- name: copy block mirror peer secret into the other cluster for replicapool
|
|
run: |
|
|
kubectl -n rook-ceph get secret pool-peer-token-replicapool -o yaml > pool-peer-token-replicapool.yaml
|
|
yq delete --inplace pool-peer-token-replicapool.yaml metadata.ownerReferences
|
|
yq write --inplace pool-peer-token-replicapool.yaml metadata.namespace rook-ceph-secondary
|
|
yq write --inplace pool-peer-token-replicapool.yaml metadata.name pool-peer-token-replicapool-config
|
|
kubectl create --namespace=rook-ceph-secondary -f pool-peer-token-replicapool.yaml
|
|
|
|
- name: copy block mirror peer secret into the other cluster for replicapool2 (using cluster global peer)
|
|
run: |
|
|
kubectl -n rook-ceph get secret cluster-peer-token-my-cluster -o yaml > cluster-peer-token-my-cluster.yaml
|
|
yq delete --inplace cluster-peer-token-my-cluster.yaml metadata.ownerReferences
|
|
yq write --inplace cluster-peer-token-my-cluster.yaml metadata.namespace rook-ceph-secondary
|
|
yq write --inplace cluster-peer-token-my-cluster.yaml metadata.name cluster-peer-token-my-cluster-config
|
|
kubectl create --namespace=rook-ceph-secondary -f cluster-peer-token-my-cluster.yaml
|
|
|
|
- name: add block mirror peer secret to the other cluster for replicapool
|
|
run: |
|
|
kubectl -n rook-ceph-secondary patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}'
|
|
|
|
- name: add block mirror peer secret to the other cluster for replicapool2 (using cluster global peer)
|
|
run: |
|
|
kubectl -n rook-ceph-secondary patch cephblockpool replicapool2 --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["cluster-peer-token-my-cluster-config"]}}}}'
|
|
|
|
- name: verify image has been mirrored for replicapool
|
|
run: |
|
|
# let's wait a bit for the image to be present
|
|
timeout 120 sh -c 'until [ "$(kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- rbd -p replicapool ls|grep -c test)" -eq 1 ]; do echo "waiting for image to be mirrored in pool replicapool" && sleep 1; done'
|
|
|
|
- name: verify image has been mirrored for replicapool2
|
|
run: |
|
|
# let's wait a bit for the image to be present
|
|
timeout 120 sh -c 'until [ "$(kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- rbd -p replicapool2 ls|grep -c test)" -eq 1 ]; do echo "waiting for image to be mirrored in pool replicapool2" && sleep 1; done'
|
|
|
|
- name: display cephblockpool and image status
|
|
run: |
|
|
timeout 80 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.mirroringStatus.summary.daemon_health}'|grep -c OK)" -eq 1 ]; do echo "waiting for mirroring status to be updated in replicapool" && sleep 1; done'
|
|
timeout 80 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool2 -o jsonpath='{.status.mirroringStatus.summary.daemon_health}'|grep -c OK)" -eq 1 ]; do echo "waiting for mirroring status to be updated in replicapool2" && sleep 1; done'
|
|
kubectl -n rook-ceph-secondary get cephblockpool replicapool -o yaml
|
|
kubectl -n rook-ceph-secondary get cephblockpool replicapool2 -o yaml
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test
|
|
|
|
- name: copy block mirror peer secret into the primary cluster for replicapool
|
|
run: |
|
|
kubectl -n rook-ceph-secondary get secret pool-peer-token-replicapool -o yaml |\
|
|
sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: pool-peer-token-replicapool/name: pool-peer-token-replicapool-config/g' |\
|
|
kubectl create --namespace=rook-ceph -f -
|
|
|
|
- name: add block mirror peer secret to the primary cluster for replicapool
|
|
run: |
|
|
kubectl -n rook-ceph patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}'
|
|
|
|
- name: wait for rook-ceph-csi-mapping-config to be updated with cluster ID
|
|
run: |
|
|
timeout 60 sh -c 'until [ "$(kubectl get cm -n rook-ceph rook-ceph-csi-mapping-config -o jsonpath='{.data.csi-mapping-config-json}' | grep -c "rook-ceph-secondary")" -eq 1 ]; do echo "waiting for rook-ceph-csi-mapping-config to be created with cluster ID mappings" && sleep 1; done'
|
|
|
|
- name: create replicated mirrored filesystem on cluster 1
|
|
run: |
|
|
PRIMARY_YAML=deploy/examples/filesystem-test-primary.yaml
|
|
cp deploy/examples/filesystem-test.yaml "$PRIMARY_YAML"
|
|
yq merge --inplace --arrays append "$PRIMARY_YAML" tests/manifests/test-fs-mirror-spec.yaml
|
|
kubectl create -f "$PRIMARY_YAML"
|
|
|
|
- name: create replicated mirrored filesystem on cluster 2
|
|
run: |
|
|
cd deploy/examples/
|
|
sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' filesystem-test.yaml
|
|
yq w -i filesystem-test.yaml spec.mirroring.enabled true
|
|
kubectl create -f filesystem-test.yaml
|
|
|
|
- name: wait for filesystem on cluster 1
|
|
run: |
|
|
timeout 300 sh -c 'until [ "$(kubectl -n rook-ceph get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done'
|
|
|
|
- name: wait for filesystem on cluster 2
|
|
run: |
|
|
timeout 300 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done'
|
|
|
|
- name: copy filesystem mirror peer secret from the secondary cluster to the primary one
|
|
run: |
|
|
kubectl -n rook-ceph-secondary get secret fs-peer-token-myfs -o yaml |\
|
|
sed '/ownerReferences/,+6d' |\
|
|
sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: fs-peer-token-myfs/name: fs-peer-token-myfs-config/g' |\
|
|
kubectl create --namespace=rook-ceph -f -
|
|
|
|
- name: add filesystem mirror peer secret to the primary cluster
|
|
run: |
|
|
kubectl -n rook-ceph patch cephfilesystem myfs --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["fs-peer-token-myfs-config"]}}}}'
|
|
|
|
- name: verify fs mirroring is working
|
|
run: |
|
|
timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -gt 3 ]; do echo "waiting for connection to peer" && sleep 1; done'
|
|
sockets=$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/)
|
|
status=$(for socket in $sockets; do minikube kubectl -- -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon /var/run/ceph/$socket help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done)
|
|
if [ "${#status}" -lt 1 ]; then echo "peer addition failed" && exit 1; fi
|
|
|
|
- name: display cephfilesystem and fs mirror daemon status
|
|
run: |
|
|
kubectl -n rook-ceph get cephfilesystem myfs -o yaml
|
|
# the check is not super ideal since 'mirroring_failed' is only displayed when there is a failure but not when it's working...
|
|
timeout 60 sh -c 'while [ "$(kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror daemon status|jq -r '.[0].filesystems[0]'|grep -c "mirroring_failed")" -eq 1 ]; do echo "waiting for filesystem to be mirrored" && sleep 1; done'
|
|
|
|
- name: Create subvolume on primary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume create myfs testsubvolume
|
|
|
|
- name: Create subvolume of same name on secondary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- ceph fs subvolume create myfs testsubvolume
|
|
|
|
- name: Deploy Direct Tools pod on primary cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/direct-mount.yaml
|
|
|
|
- name: Deploy Direct Tools pod on secondary cluster
|
|
run: |
|
|
sed -i "s/rook-ceph # namespace/rook-ceph-secondary # namespace/" deploy/examples/direct-mount.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/direct-mount.yaml
|
|
|
|
- name: Configure a directory for snapshot mirroring on primary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror enable myfs
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror add myfs /volumes/_nogroup/testsubvolume/
|
|
|
|
- name: make sure that snapshot mirror is enabled on the secondary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- ceph fs snapshot mirror enable myfs
|
|
|
|
- name: Create 3 snapshots on cluster primary cluster
|
|
run: |
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume snapshot create myfs testsubvolume snap1
|
|
kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume snapshot create myfs testsubvolume snap2
|
|
|
|
- name: Get the peer and verify the peer synchronization status that snaps have synced on secondary cluster
|
|
run: |
|
|
exec_fs_mirror='kubectl -n rook-ceph exec deploy/rook-ceph-fs-mirror --'
|
|
mirror_daemon=$($exec_fs_mirror ls /var/run/ceph/ | grep "fs-mirror" | head -n 1)
|
|
# timeout 45 bash -x <<EOF
|
|
# while
|
|
# clusterfsid=\$($exec_fs_mirror ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror status myfs@1 |jq -r '.peers|keys[]')
|
|
# [ -z "\$clusterfsid" ]
|
|
# do echo "Waiting for the clusterfsid to get populated." && sleep 1
|
|
# done
|
|
# EOF
|
|
# clusterfsid=$($exec_fs_mirror ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror status myfs@1 |jq -r '.peers|keys[]')
|
|
# echo $clusterfsid
|
|
# kubectl -n rook-ceph-secondary wait pod -l app=rook-direct-mount --for condition=Ready --timeout=400s
|
|
# kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- mkdir /tmp/registry
|
|
# mon_endpoints=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- grep mon_host /etc/ceph/ceph.conf | awk '{print $3}')
|
|
# my_secret=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- grep key /etc/ceph/keyring | awk '{print $3}')
|
|
# kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- mount -t ceph -o mds_namespace=myfs,name=admin,secret=$my_secret $mon_endpoints:/ /tmp/registry
|
|
# num_snaps_target=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- ls -lhsa /tmp/registry/volumes/_nogroup/testsubvolume/.snap|grep snap|wc -l)
|
|
# snaps=$(kubectl -n rook-ceph exec deploy/rook-ceph-fs-mirror -- ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror peer status myfs@1 $clusterfsid|jq -r '."/volumes/_nogroup/testsubvolume"."snaps_synced"')
|
|
# echo "snapshots: $snaps"
|
|
# if [ $num_snaps_target = $snaps ]
|
|
# then echo "Snapshots have synced."
|
|
# else echo "Snapshots have not synced. NEEDS INVESTIGATION"
|
|
# fi
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
additional-namespace: rook-ceph-secondary
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
rgw-multisite-testing:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: install additional deps for object testing
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
sudo apt-get install -y s3cmd
|
|
|
|
- name: use local disk into two partitions
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2
|
|
sudo lsblk
|
|
|
|
- name: deploy first cluster rook
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_first_rook_cluster
|
|
kubectl create -f deploy/examples/object-multisite-test.yaml
|
|
# wait for multisite-store to be created
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectstore multisite-store rook-ceph 600
|
|
|
|
- name: prep second cluster pull realm config
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
cd deploy/examples/
|
|
IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
|
|
yq w -i -d1 object-multisite-pull-realm-test.yaml spec.pull.endpoint http://${IP_ADDR}:80
|
|
BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
|
|
BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}")
|
|
sed -i 's/VzFjNFltMVdWRTFJWWxZelZWQT0=/'"$BASE64_ACCESS_KEY"'/g' object-multisite-pull-realm-test.yaml
|
|
sed -i 's/WVY1MFIxeExkbG84U3pKdlRseEZXVGR3T3k1U1dUSS9KaTFoUVE9PQ==/'"$BASE64_SECRET_KEY"'/g' object-multisite-pull-realm-test.yaml
|
|
|
|
- name: deploy second cluster rook
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_second_rook_cluster
|
|
kubectl create -f deploy/examples/object-multisite-pull-realm-test.yaml
|
|
# wait for realms to be pulled and zone-b-multisite-store to be created
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectstore zone-b-multisite-store rook-ceph-secondary 600
|
|
|
|
- name: wait for both ceph clusters to be ready
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph
|
|
tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph-secondary
|
|
|
|
- name: write an object to one cluster, read from the other
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
tests/scripts/github-action-helper.sh test_multisite_object_replication
|
|
|
|
# if this test fails, it could mean the RGW `period get` or `period update` output has changed
|
|
- name: RGW configuration period should be committed on first reconcile and not be committed on second reconcile
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
ns_name_primary_object_store='"rook-ceph/multisite-store"' # double quotes intended
|
|
ns_name_secondary_object_store='"rook-ceph-secondary/zone-b-multisite-store"' # double quotes intended
|
|
committed_msg="committing changes to RGW configuration period for CephObjectStore"
|
|
tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_primary_object_store}"
|
|
tests/scripts/github-action-helper.sh verify_operator_log_message "${committed_msg} ${ns_name_secondary_object_store}"
|
|
ns_name_primary_object_zone='"rook-ceph/zone-a"' # double quotes intended
|
|
ns_name_secondary_object_zone='"rook-ceph-secondary/zone-b"' # double quotes intended
|
|
tests/scripts/github-action-helper.sh restart_operator
|
|
not_committed_msg="there are no changes to commit for RGW configuration period for CephObjectStore"
|
|
tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_primary_object_zone}" 600
|
|
tests/scripts/github-action-helper.sh wait_for_operator_log_message "${not_committed_msg} ${ns_name_secondary_object_zone}" 600
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
additional-namespace: rook-ceph-secondary
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
nvmeof-protocol:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ["quay.io/ceph/ceph:v20"]
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: install nvme initiator prerequisites
|
|
run: tests/scripts/github-action-helper.sh install_nvme_initiator_prerequisites
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/operator.yaml"
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/csi-operator.yaml"
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/csi/nvmeof/driver.yaml"
|
|
|
|
export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
yq w -i -d0 deploy/examples/cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"
|
|
kubectl create -f deploy/examples/cluster-test.yaml
|
|
tests/scripts/github-action-helper.sh deploy_toolbox
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
env:
|
|
CSI_POD_MIN_COUNT: "6"
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
|
|
- name: test ceph-csi-nvmeof plugin restart and data persistence
|
|
run: tests/scripts/github-action-helper.sh test_csi_nvmeof_workload
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
encryption-pvc-kms-ibm-kp:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: run encryption KMS IBM Key Protect
|
|
uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
|
|
if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''"
|
|
env:
|
|
IBM_KP_SERVICE_INSTANCE_ID: ${{ secrets.IBM_INSTANCE_ID }}
|
|
IBM_KP_SERVICE_API_KEY: ${{ secrets.IBM_SERVICE_API_KEY }}
|
|
with:
|
|
ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }}
|
|
ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
artifact-name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
ceph-image: ${{ matrix.ceph-image }}
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
multus-public-and-cluster:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-multus-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: Setup multus
|
|
run: ./tests/scripts/multus/setup-multus.sh
|
|
|
|
- name: Set up multus prerequisite host routing
|
|
run: kubectl create -f tests/scripts/multus/host-cfg-ds.yaml
|
|
|
|
- name: create cluster prerequisites
|
|
run: tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
|
|
|
- name: Install public and cluster NADs in rook-ceph namespace
|
|
run: kubectl create -f tests/scripts/multus/rook-ceph-public-cluster-nads.yaml
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
|
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
|
|
|
|
|
|
export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
|
yq write -i deploy/examples/cluster-multus-test.yaml "spec.storage.deviceFilter" "${BLOCK}"
|
|
kubectl create -f deploy/examples/cluster-multus-test.yaml
|
|
|
|
tests/scripts/github-action-helper.sh deploy_toolbox
|
|
|
|
kubectl create -f deploy/examples/filesystem-test.yaml
|
|
kubectl create -f deploy/examples/nfs-test.yaml
|
|
|
|
- name: wait for prepare pod
|
|
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
|
|
|
|
- name: wait for ceph to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
|
|
|
- name: wait for cephnfs to be ready
|
|
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready nfs 1
|
|
|
|
- name: check multus connections
|
|
run: tests/scripts/github-action-helper.sh test_multus_connections
|
|
|
|
- name: test ceph-csi-rbd plugin restart
|
|
run: tests/scripts/github-action-helper.sh test_csi_rbd_workload
|
|
|
|
- name: test ceph-csi-cephfs plugin restart
|
|
run: tests/scripts/github-action-helper.sh test_csi_cephfs_workload
|
|
|
|
- name: test ceph-csi-nfs plugin restart
|
|
run: tests/scripts/github-action-helper.sh test_csi_nfs_workload
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
additional-namespace: kube-system
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
two-object-one-zone:
|
|
runs-on: ubuntu-22.04
|
|
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
|
|
strategy:
|
|
matrix:
|
|
ceph-image: ${{ fromJson(inputs.ceph_images) }}
|
|
kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: consider (pre-job) debugging
|
|
uses: ./.github/workflows/tmate_debug
|
|
with:
|
|
use-tmate: ${{ secrets.USE_TMATE }}
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|
|
|
|
- name: setup cluster resources
|
|
uses: ./.github/workflows/integration-test-setup-cluster-resources
|
|
with:
|
|
kubernetes-version: ${{ matrix.kubernetes-version }}
|
|
|
|
- name: set Ceph version in CephCluster manifest
|
|
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
|
|
|
|
- name: validate-yaml
|
|
run: tests/scripts/github-action-helper.sh validate_yaml
|
|
|
|
- name: use local disk and create partitions for osds
|
|
run: |
|
|
tests/scripts/github-action-helper.sh use_local_disk
|
|
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
|
|
|
- name: deploy cluster
|
|
run: |
|
|
tests/scripts/github-action-helper.sh deploy_cluster
|
|
tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph 240
|
|
|
|
- name: create CephBlockPool(s), CephObjectZone, and CephObjectStore(s)
|
|
shell: bash --noprofile --norc -eo pipefail -x {0}
|
|
run: |
|
|
kubectl create -f deploy/examples/object-separate-pools-test.yaml
|
|
kubectl create -f deploy/examples/two-object-one-zone-test.yaml
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectzone object-separate-pools rook-ceph 600
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectstore two-object-one-zone-alpha rook-ceph 600
|
|
tests/scripts/github-action-helper.sh wait_for cephobjectstore two-object-one-zone-beta rook-ceph 600
|
|
|
|
- name: check for pools created by RGW that are unexpected
|
|
run: tests/scripts/github-action-helper.sh test_object_separate_pools
|
|
|
|
- name: verify that only one zone exists
|
|
run: |
|
|
[[ $(tests/scripts/github-action-helper.sh radosgw-admin zone list | jq '.zones | length') == "1" ]]
|
|
|
|
- name: collect common logs
|
|
if: always()
|
|
uses: ./.github/workflows/collect-logs
|
|
with:
|
|
name: ${{ github.job }}-${{ matrix.ceph-image }}
|
|
|
|
- name: consider (post-job) debugging
|
|
uses: ./.github/workflows/upterm_debug
|
|
with:
|
|
debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
|