diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 97ef1c7b8..09f34e6c5 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1957,6 +1957,78 @@ jobs: with: debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }} + nvmeof-protocol: + runs-on: ubuntu-22.04 + if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" + strategy: + matrix: + ceph-image: ["quay.io/ceph/ceph:v20"] + kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }} + steps: + - name: checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: consider (pre-job) debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }} + + - name: setup cluster resources + uses: ./.github/workflows/integration-test-setup-cluster-resources + with: + kubernetes-version: ${{ matrix.kubernetes-version }} + + - name: install nvme initiator prerequisites + run: tests/scripts/github-action-helper.sh install_nvme_initiator_prerequisites + + - name: set Ceph version in CephCluster manifest + run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}" + + - name: validate-yaml + run: tests/scripts/github-action-helper.sh validate_yaml + + - name: use local disk and create partitions for osds + run: | + tests/scripts/github-action-helper.sh use_local_disk + tests/scripts/github-action-helper.sh create_partitions_for_osds + + - name: deploy cluster without ceph-csi operator + run: | + OPERATOR_MANIFEST="deploy/examples/operator.yaml" + sed -i 's|# ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.16.1"|ROOK_CSI_CEPH_IMAGE: "quay.io/nixpanic/cephcsi:nvmeof"|' "${OPERATOR_MANIFEST}" + sed -i 's|ROOK_USE_CSI_OPERATOR: "true"|ROOK_USE_CSI_OPERATOR: "false"|' "${OPERATOR_MANIFEST}" + + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "${OPERATOR_MANIFEST}" + + export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)" + yq w -i -d0 deploy/examples/cluster-test.yaml spec.storage.deviceFilter "${BLOCK}" + kubectl create -f deploy/examples/cluster-test.yaml + tests/scripts/github-action-helper.sh deploy_toolbox + + - name: wait for prepare pod + run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 + + - name: wait for ceph to be ready + env: + CSI_POD_MIN_COUNT: "6" + run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 + + - name: test ceph-csi-nvmeof plugin restart and data persistence + run: tests/scripts/github-action-helper.sh test_csi_nvmeof_workload + + - name: collect common logs + if: always() + uses: ./.github/workflows/collect-logs + with: + name: ${{ github.job }}-${{ matrix.ceph-image }} + - name: consider (post-job) debugging + uses: ./.github/workflows/upterm_debug + with: + debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }} + encryption-pvc-kms-ibm-kp: runs-on: ubuntu-22.04 if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" diff --git a/deploy/examples/csi/nvmeof/storageclass.yaml b/deploy/examples/csi/nvmeof/storageclass.yaml index 328c435c4..2f6691ffd 100644 --- a/deploy/examples/csi/nvmeof/storageclass.yaml +++ b/deploy/examples/csi/nvmeof/storageclass.yaml @@ -14,8 +14,6 @@ parameters: listeners: | [ { - "address": "10.129.2.36", - "port": 4420, "hostname": "rook-ceph-nvmeof-nvmeof-a" } ] diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 14ff11e0c..da5ae8a7e 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -83,6 +83,18 @@ function install_deps() { sudo chmod +x /usr/local/bin/yq } +function install_nvme_initiator_prerequisites() { + sudo apt-get update + sudo apt-get install -y nvme-cli + sudo apt-get install -y "linux-modules-extra-$(uname -r)" || sudo apt-get install -y linux-modules-extra-azure || true + sudo modprobe nvme_fabrics || true + sudo modprobe nvme_tcp || true + if [[ ! -d /sys/module/nvme_tcp ]]; then + echo "nvme_tcp kernel module is unavailable on this runner kernel: $(uname -r)" + exit 1 + fi +} + function print_k8s_cluster_status() { kubectl cluster-info kubectl get pods -n kube-system @@ -740,6 +752,99 @@ function test_csi_nfs_workload { kubectl exec -t pod/csinfs-demo-pod -- ls -alh /var/lib/www/html/ } +function test_csi_nvmeof_workload { + cd "${REPO_DIR}/deploy/examples/csi/nvmeof" + + local cephcsi_image="${1:-quay.io/nixpanic/cephcsi:nvmeof}" + local gateway_svc="rook-ceph-nvmeof-nvmeof-a" + local gateway_listener_hostname="rook-ceph-nvmeof-nvmeof-a" + local service_ip + local old_gateway_pod + + sed -i 's/failureDomain: .*/failureDomain: osd/' nvmeof-pool.yaml + sed -i 's/size: .*/size: 1/' nvmeof-pool.yaml + kubectl create -f nvmeof-pool.yaml + wait_for cephblockpool nvmeof rook-ceph 600 + + kubectl create -f "${REPO_DIR}/deploy/examples/nvmeof-test.yaml" + wait_for cephnvmeofgateway nvmeof rook-ceph 600 + timeout 300 bash < /mnt/nvmeof/test.txt && cat /mnt/nvmeof/test.txt" + kubectl -n default exec pod/nvmeof-test-pod -- sh -c "grep -qx 'abcd' /mnt/nvmeof/test.txt" + + old_gateway_pod="$(kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ {print $1; exit}')" + kubectl -n rook-ceph delete pod "${old_gateway_pod}" + timeout 300 bash < /mnt/nvmeof/test.txt && grep -qx 'abcd efg' /mnt/nvmeof/test.txt" + + kubectl delete pod nvmeof-test-pod --grace-period=0 --force + kubectl wait --for=delete pod/nvmeof-test-pod --timeout=180s + + kubectl create -f pod.yaml + kubectl wait pod/nvmeof-test-pod --for=condition=Ready --timeout=300s + kubectl -n default exec pod/nvmeof-test-pod -- sh -c "grep -qx 'abcd efg' /mnt/nvmeof/test.txt" + kubectl -n default exec pod/nvmeof-test-pod -- sh -c "echo 'abcd efg hij' > /mnt/nvmeof/test.txt && grep -qx 'abcd efg hij' /mnt/nvmeof/test.txt" +} + function toolbox() { kubectl -n rook-ceph exec -it "$(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}')" -- "$@" } diff --git a/tests/scripts/validate_cluster.sh b/tests/scripts/validate_cluster.sh index eb6c3cd24..b61eefcee 100755 --- a/tests/scripts/validate_cluster.sh +++ b/tests/scripts/validate_cluster.sh @@ -102,9 +102,10 @@ function test_demo_pool { } function test_csi { - timeout 360 bash -x <<-'EOF' - until [[ "$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running|grep -c csi.)" -ge 7 ]]; do - echo "waiting for csi pods to be ready" + csi_pod_min_count="${CSI_POD_MIN_COUNT:-7}" + timeout 360 bash -x <