nvmeof: add nvmeof minikube canary test without csi operator

adds a new nvmeof minikube canary job for ceph v20.
the test deploys rook with csi operator disabled for nvmeof flow.
it validates pvc and pod io, gateway restart, and data persistence.

Signed-off-by: Oded Viner <oviner@redhat.com>
(cherry picked from commit 2ebb5d968a)

.github/workflows/canary-integration-test.yml

@@ -1957,6 +1957,78 @@ jobs:
         with:
           debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
 
+  nvmeof-protocol:
+    runs-on: ubuntu-22.04
+    if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
+    strategy:
+      matrix:
+        ceph-image: ["quay.io/ceph/ceph:v20"]
+        kubernetes-version: ${{ fromJson(inputs.kubernetes-version) }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+
+      - name: consider (pre-job) debugging
+        uses: ./.github/workflows/tmate_debug
+        with:
+          use-tmate: ${{ secrets.USE_TMATE }}
+          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
+
+      - name: setup cluster resources
+        uses: ./.github/workflows/integration-test-setup-cluster-resources
+        with:
+          kubernetes-version: ${{ matrix.kubernetes-version }}
+
+      - name: install nvme initiator prerequisites
+        run: tests/scripts/github-action-helper.sh install_nvme_initiator_prerequisites
+
+      - name: set Ceph version in CephCluster manifest
+        run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}"
+
+      - name: validate-yaml
+        run: tests/scripts/github-action-helper.sh validate_yaml
+
+      - name: use local disk and create partitions for osds
+        run: |
+          tests/scripts/github-action-helper.sh use_local_disk
+          tests/scripts/github-action-helper.sh create_partitions_for_osds
+
+      - name: deploy cluster without ceph-csi operator
+        run: |
+          OPERATOR_MANIFEST="deploy/examples/operator.yaml"
+          sed -i 's|# ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.16.1"|ROOK_CSI_CEPH_IMAGE: "quay.io/nixpanic/cephcsi:nvmeof"|' "${OPERATOR_MANIFEST}"
+          sed -i 's|ROOK_USE_CSI_OPERATOR: "true"|ROOK_USE_CSI_OPERATOR: "false"|' "${OPERATOR_MANIFEST}"
+
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "${OPERATOR_MANIFEST}"
+
+          export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
+          yq w -i -d0 deploy/examples/cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"
+          kubectl create -f deploy/examples/cluster-test.yaml
+          tests/scripts/github-action-helper.sh deploy_toolbox
+
+      - name: wait for prepare pod
+        run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
+
+      - name: wait for ceph to be ready
+        env:
+          CSI_POD_MIN_COUNT: "6"
+        run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
+
+      - name: test ceph-csi-nvmeof plugin restart and data persistence
+        run: tests/scripts/github-action-helper.sh test_csi_nvmeof_workload
+
+      - name: collect common logs
+        if: always()
+        uses: ./.github/workflows/collect-logs
+        with:
+          name: ${{ github.job }}-${{ matrix.ceph-image }}
+      - name: consider (post-job) debugging
+        uses: ./.github/workflows/upterm_debug
+        with:
+          debug-ci: ${{ contains(github.event.pull_request.labels.*.name, 'debug-ci') }}
+
   encryption-pvc-kms-ibm-kp:
     runs-on: ubuntu-22.04
     if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"

deploy/examples/csi/nvmeof/storageclass.yaml

@@ -14,8 +14,6 @@ parameters:
   listeners: |
     [
       {
-        "address": "10.129.2.36",
-        "port": 4420,
         "hostname": "rook-ceph-nvmeof-nvmeof-a"
       }
     ]

tests/scripts/github-action-helper.sh

@@ -83,6 +83,18 @@ function install_deps() {
   sudo chmod +x /usr/local/bin/yq
 }
 
+function install_nvme_initiator_prerequisites() {
+  sudo apt-get update
+  sudo apt-get install -y nvme-cli
+  sudo apt-get install -y "linux-modules-extra-$(uname -r)" || sudo apt-get install -y linux-modules-extra-azure || true
+  sudo modprobe nvme_fabrics || true
+  sudo modprobe nvme_tcp || true
+  if [[ ! -d /sys/module/nvme_tcp ]]; then
+    echo "nvme_tcp kernel module is unavailable on this runner kernel: $(uname -r)"
+    exit 1
+  fi
+}
+
 function print_k8s_cluster_status() {
   kubectl cluster-info
   kubectl get pods -n kube-system
@@ -740,6 +752,99 @@ function test_csi_nfs_workload {
   kubectl exec -t pod/csinfs-demo-pod -- ls -alh /var/lib/www/html/
 }
 
+function test_csi_nvmeof_workload {
+  cd "${REPO_DIR}/deploy/examples/csi/nvmeof"
+
+  local cephcsi_image="${1:-quay.io/nixpanic/cephcsi:nvmeof}"
+  local gateway_svc="rook-ceph-nvmeof-nvmeof-a"
+  local gateway_listener_hostname="rook-ceph-nvmeof-nvmeof-a"
+  local service_ip
+  local old_gateway_pod
+
+  sed -i 's/failureDomain: .*/failureDomain: osd/' nvmeof-pool.yaml
+  sed -i 's/size: .*/size: 1/' nvmeof-pool.yaml
+  kubectl create -f nvmeof-pool.yaml
+  wait_for cephblockpool nvmeof rook-ceph 600
+
+  kubectl create -f "${REPO_DIR}/deploy/examples/nvmeof-test.yaml"
+  wait_for cephnvmeofgateway nvmeof rook-ceph 600
+  timeout 300 bash <<EOF
+until kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ {print \$3}' | grep -q "^Running$"; do
+  echo "waiting for NVMe-oF gateway pod to be running"
+  sleep 5
+done
+EOF
+
+  service_ip="$(kubectl -n rook-ceph get svc "${gateway_svc}" -o jsonpath='{.spec.clusterIP}')"
+  if [[ -z "${service_ip}" ]]; then
+    echo "failed to discover gateway service IP for ${gateway_svc}"
+    exit 1
+  fi
+
+  sed -i "s|nvmeofGatewayAddress: \".*\"|nvmeofGatewayAddress: \"${service_ip}\"|g" storageclass.yaml
+  # ceph-csi PR 6061 expects listeners to include hostname only.
+  sed -i '/"address":/d' storageclass.yaml
+  sed -i '/"port":/d' storageclass.yaml
+  sed -i "s|\"hostname\": \".*\"|\"hostname\": \"${gateway_listener_hostname}\"|g" storageclass.yaml
+  sed -i "s|image: quay.io/cephcsi/cephcsi:.*|image: ${cephcsi_image}|g" provisioner.yaml
+  sed -i "s|image: quay.io/cephcsi/cephcsi:.*|image: ${cephcsi_image}|g" node-plugin.yaml
+
+  kubectl create -f provisioner.yaml
+  kubectl -n rook-ceph wait --for=condition=available deployment/csi-nvmeofplugin-provisioner --timeout=300s
+
+  kubectl create -f storageclass.yaml
+  kubectl create -f pvc.yaml
+  if ! kubectl wait pvc/nvmeof-external-volume --for=jsonpath='{.status.phase}'=Bound --timeout=300s; then
+    echo "PVC nvmeof-external-volume did not reach Bound; collecting diagnostics..."
+    kubectl describe pvc nvmeof-external-volume || true
+    kubectl get events -n default --sort-by=.metadata.creationTimestamp | tail -n 50 || true
+    kubectl -n rook-ceph logs deploy/csi-nvmeofplugin-provisioner -c csi-provisioner --tail=200 || true
+    kubectl -n rook-ceph logs deploy/csi-nvmeofplugin-provisioner -c csi-nvmeofplugin --tail=200 || true
+    return 1
+  fi
+
+  # Sanity check module availability; prerequisites are installed earlier in CI setup.
+  lsmod | grep -E 'nvme(_|-)(tcp|fabrics)' || true
+  if [[ ! -d /sys/module/nvme_tcp ]]; then
+    echo "nvme_tcp kernel module is required but unavailable on kernel $(uname -r)"
+    return 1
+  fi
+
+  kubectl create -f node-plugin.yaml
+  if ! kubectl -n rook-ceph rollout status daemonset/nvmeof.csi.ceph.com-nodeplugin --timeout=300s; then
+    kubectl -n rook-ceph get pods -l app=nvmeof.csi.ceph.com-nodeplugin -o wide || true
+    kubectl -n rook-ceph describe daemonset nvmeof.csi.ceph.com-nodeplugin || true
+    echo "nvmeof nodeplugin daemonset failed to become ready"
+    return 1
+  fi
+
+  kubectl create -f pod.yaml
+  kubectl wait pod/nvmeof-test-pod --for=condition=Ready --timeout=300s
+
+  kubectl -n default exec pod/nvmeof-test-pod -- sh -c "echo 'abcd' > /mnt/nvmeof/test.txt && cat /mnt/nvmeof/test.txt"
+  kubectl -n default exec pod/nvmeof-test-pod -- sh -c "grep -qx 'abcd' /mnt/nvmeof/test.txt"
+
+  old_gateway_pod="$(kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ {print $1; exit}')"
+  kubectl -n rook-ceph delete pod "${old_gateway_pod}"
+  timeout 300 bash <<EOF
+until new_gateway_pod=\$(kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ && \$3 == "Running" {print \$1; exit}') && [[ -n "\${new_gateway_pod}" && "\${new_gateway_pod}" != "${old_gateway_pod}" ]]; do
+  echo "waiting for NVMe-oF gateway pod restart"
+  sleep 5
+done
+EOF
+
+  kubectl -n default exec pod/nvmeof-test-pod -- sh -c "grep -qx 'abcd' /mnt/nvmeof/test.txt"
+  kubectl -n default exec pod/nvmeof-test-pod -- sh -c "echo 'abcd efg' > /mnt/nvmeof/test.txt && grep -qx 'abcd efg' /mnt/nvmeof/test.txt"
+
+  kubectl delete pod nvmeof-test-pod --grace-period=0 --force
+  kubectl wait --for=delete pod/nvmeof-test-pod --timeout=180s
+
+  kubectl create -f pod.yaml
+  kubectl wait pod/nvmeof-test-pod --for=condition=Ready --timeout=300s
+  kubectl -n default exec pod/nvmeof-test-pod -- sh -c "grep -qx 'abcd efg' /mnt/nvmeof/test.txt"
+  kubectl -n default exec pod/nvmeof-test-pod -- sh -c "echo 'abcd efg hij' > /mnt/nvmeof/test.txt && grep -qx 'abcd efg hij' /mnt/nvmeof/test.txt"
+}
+
 function toolbox() {
   kubectl -n rook-ceph exec -it "$(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}')" -- "$@"
 }

tests/scripts/validate_cluster.sh

@@ -102,9 +102,10 @@ function test_demo_pool {
 }
 
 function test_csi {
-  timeout 360 bash -x <<-'EOF'
-    until [[ "$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running|grep -c csi.)" -ge 7 ]]; do
-      echo "waiting for csi pods to be ready"
+  csi_pod_min_count="${CSI_POD_MIN_COUNT:-7}"
+  timeout 360 bash -x <<EOF
+    until [[ "\$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running | grep -c csi.)" -ge "${csi_pod_min_count}" ]]; do
+      echo "waiting for csi pods to be ready (need ${csi_pod_min_count})"
       kubectl -n rook-ceph get pods | grep csi
       sleep 5
     done