From 8db438b8e39f994d08780d62b148152753ae5514 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Wed, 1 Jul 2026 14:13:03 -0700 Subject: [PATCH] ci: convert integration and canary CI from minikube to kind The integration and canary suites ran on a single-node minikube `driver: none` cluster, where the kubelet ran directly on the GitHub runner, so the host docker daemon doubled as the cluster runtime and host block devices and host paths were directly visible to pods. Replace that with a kind cluster. Every suite creates its cluster through the shared integration-test-setup-cluster-resources composite action, so the conversion is centralized there and converts the smoke, object, helm, keystone, multi-cluster, upgrade, on-release, nightly, encryption-KMS and all canary jobs at once. - Replace the setup-minikube step with helm/kind-action, selecting the kubernetes version via the kindest/node image tag and creating a single-node cluster from a new kind config (kind pinned to v0.32.0 for reproducibility). - Drop the cri-dockerd install; kind nodes use their built-in containerd. - Add a kind config that bind-mounts the host /dev, /var/lib/rook and /run/udev into the node so the existing host-based disk-prep helpers (use_local_disk*, create_partitions_for_osds, blockDevicePV.sh, localPathPV.sh, ...) keep working unchanged: devices and partitions created on the host appear in the node and in the OSD pods that hostPath-mount the node /dev, and ceph-volume can read the host udev database it needs to inventory disks. - Prepare the kind node for the host-level operations rook runs against the underlying host: remount /sys read-write so CSI's kernel RBD mapping (`rbd map --device-type krbd`, which writes /sys/bus/rbd) works, and install lvm2 and cryptsetup, which rook runs in the node's mount namespace to provision LVM- and encryption-backed OSDs. kindest/node images provide none of this; the minikube driver:none runner host did. - Route the Service and pod CIDRs from the runner to the kind node so host-side tests (the `go test` process runs on the runner) can reach in-cluster ClusterIPs, e.g. an S3 request to the RGW service. With minikube driver:none the runner already shared the cluster network. - Load locally built images into the cluster. Under minikube `driver: none` the built image was already in the cluster runtime; under kind it must be imported, so build_rook and create_helm_tag now import their images into each node's containerd through a new load_image_into_cluster helper (via the node's ctr, which avoids the kind/kindest-node containerd-config version skew that breaks `kind load docker-image`). - Point Vault's kubernetes-auth at the in-cluster API endpoint (kubernetes.default.svc) instead of the kubeconfig server URL: kind exposes that as https://127.0.0.1:, unreachable from the in-cluster Vault pod, so OSD encryption-key retrieval via k8s-auth failed. - Replace the remaining direct minikube references in the canary workflow: a `minikube kubectl` call and the external-cluster topology values. - Adapt host-name assumptions that only held under driver:none: resolve the disk-cleanup job by the k8s node name rather than the runner hostname, and let kind-action ignore post-job cluster-teardown failures (the runner is ephemeral; nvme/multus devices can wedge `docker rm` of the node). - Update stale comments that described the CI environment as minikube. - Move the multus integration test's kind config under tests/config too, so both kind cluster configs live in one place. create-dev-cluster.sh and other local-dev tooling are intentionally left on minikube. Signed-off-by: Joshua Hoblitt (cherry picked from commit d59abb195608294bba715f80cf31d3af1b6abd55) --- .github/workflows/canary-integration-test.yml | 8 +- .../action.yaml | 52 +++++------- .github/workflows/multus.yaml | 2 +- tests/config/kind-config.yaml | 29 +++++++ .../multus/kind-config.yaml | 0 tests/framework/installer/ceph_manifests.go | 2 +- tests/scripts/deploy-validate-vault.sh | 8 +- tests/scripts/github-action-helper.sh | 81 ++++++++++++++++++- tests/scripts/multus/host-cfg-ds.yaml | 2 +- tests/scripts/multus/test-110-cli.sh | 2 +- .../multus/test-210-stretch-overlap.sh | 2 +- 11 files changed, 143 insertions(+), 45 deletions(-) create mode 100644 tests/config/kind-config.yaml rename tests/{scripts => config}/multus/kind-config.yaml (100%) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index c6805e048..ea7854c0e 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -280,15 +280,15 @@ jobs: kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1c -nrook-ceph # pass correct flags - kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03 + kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values kind-control-plane,arbitrary,value # pass the pool which is not exists - if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then + if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values kind-control-plane,arbitrary,value); then echo "script run completed with stderr error after passing the wrong pools: $output" else echo "script failed because wrong pools doesn't exist" fi # dont pass all topology flags - if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then + if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values kind-control-plane,arbitrary,value); then echo "script run completed with stderr error after passing the wrong flags: $output" else echo "script failed because topology-failure-domain-label is missing" @@ -1831,7 +1831,7 @@ jobs: timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -gt 3 ]; do echo "waiting for connection to peer" && sleep 1; done' sockets=$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/) # shellcheck disable=SC2086 # word splitting of $sockets into separate socket paths is intentional - status=$(for socket in $sockets; do minikube kubectl -- -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon "/var/run/ceph/$socket" help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done) + status=$(for socket in $sockets; do kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon "/var/run/ceph/$socket" help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done) if [ "${#status}" -lt 1 ]; then echo "peer addition failed" && exit 1; fi - name: display cephfilesystem and fs mirror daemon status diff --git a/.github/workflows/integration-test-setup-cluster-resources/action.yaml b/.github/workflows/integration-test-setup-cluster-resources/action.yaml index 9f38b6da2..2cff5beae 100644 --- a/.github/workflows/integration-test-setup-cluster-resources/action.yaml +++ b/.github/workflows/integration-test-setup-cluster-resources/action.yaml @@ -22,39 +22,27 @@ runs: with: go-version: "1.26" - - name: Install cri-dockerd - shell: bash --noprofile --norc -eo pipefail -x {0} - run: | - ARCH=$(go env GOARCH) - # --fail so an HTTP error page is not saved as the .deb (dpkg then fails on the - # corrupt archive), and retry transient download errors - curl -fLO --retry 5 --retry-all-errors --retry-delay 10 https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.21/cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb - sudo dpkg -i cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb - rm -f cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb - - - name: Setup Minikube - uses: medyagh/setup-minikube@latest + - name: create kind cluster + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 with: - # lock minikube version to ensure release branch CI doesn't fail when latest minikube no - # longer supports older k8s version we lock to - minikube-version: "1.38.0" - kubernetes-version: ${{ inputs.kubernetes-version }} - driver: none - container-runtime: docker - memory: 6g - cpus: 2 - addons: ingress - cni: calico - # --force: when the requested kubernetes version is missing from minikube's compiled-in - # version list (any version released after the pinned minikube), 'minikube start' - # verifies the version with an anonymous GitHub API request, which is regularly - # rate-limited on shared runner IPs and fails the job (K8S_FAIL_CONNECT, exit code 40). - # The versions used here are pinned constants already validated by the PRs that bump - # them, so that check adds nothing in CI; an invalid version would still fail fast at - # the kubeadm/kubelet download. - # --wait-timeout: minikube's default 6m node-ready wait expires on slow runners - # (GUEST_START, exit code 80). - start-args: --force --wait-timeout=15m + # Pin kind; its release sets which kindest/node image versions are available. + version: v0.32.0 + # Each matrix kubernetes-version needs a published kindest/node: image. + node_image: kindest/node:${{ inputs.kubernetes-version }} + # See the config; it bind-mounts the host paths the disk-prep helpers need. + config: tests/config/kind-config.yaml + cluster_name: kind + wait: 300s + # Runner is ephemeral; some jobs hold node devices via the /dev mount, wedging teardown. + ignore_failed_clean: "true" + + - name: prepare the kind node + shell: bash --noprofile --norc -eo pipefail -x {0} + run: tests/scripts/github-action-helper.sh prepare_kind_node + + - name: route host to cluster services + shell: bash --noprofile --norc -eo pipefail -x {0} + run: tests/scripts/github-action-helper.sh add_host_routes_to_cluster - name: install deps shell: bash --noprofile --norc -eo pipefail -x {0} diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml index 7db5749a0..c35b89ecb 100644 --- a/.github/workflows/multus.yaml +++ b/.github/workflows/multus.yaml @@ -46,7 +46,7 @@ jobs: - name: Create KinD Cluster uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 with: - config: tests/scripts/multus/kind-config.yaml + config: tests/config/multus/kind-config.yaml cluster_name: kind - name: consider (pre-job) debugging diff --git a/tests/config/kind-config.yaml b/tests/config/kind-config.yaml new file mode 100644 index 000000000..3494c2227 --- /dev/null +++ b/tests/config/kind-config.yaml @@ -0,0 +1,29 @@ +# Kind cluster configuration for Rook's canary and integration test CI. +# +# Kind runs the node as a container, so host resources are not visible to it by default. The +# extraMounts below bind-mount the resources the host-based disk-prep helpers (use_local_disk*, +# create_partitions_for_osds, blockDevicePV.sh, localPathPV.sh, ...) depend on: +# - /dev : block devices, partitions and LVM devices created on the host +# appear inside the node (shared devtmpfs) and, in turn, inside the OSD +# pods, which hostPath-mount the node's /dev. +# - /var/lib/rook : dataDirHostPath and the local-PV mon directories created on the host. +# - /run/udev : ceph-volume reads the udev database (/run/udev/data) to inventory a +# disk; the kind node runs no udevd, so without the host's populated db +# ceph-volume fails with "No udev data could be retrieved" and rook skips +# the device. The runner host runs udevd, so bind-mount its db. +# HostToContainer propagation ensures devices created on the host after the cluster is up +# (e.g. partitions and extra iSCSI disks set up by per-job steps) also appear inside the node. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraMounts: + - hostPath: /dev + containerPath: /dev + propagation: HostToContainer + - hostPath: /var/lib/rook + containerPath: /var/lib/rook + propagation: HostToContainer + - hostPath: /run/udev + containerPath: /run/udev + propagation: HostToContainer diff --git a/tests/scripts/multus/kind-config.yaml b/tests/config/multus/kind-config.yaml similarity index 100% rename from tests/scripts/multus/kind-config.yaml rename to tests/config/multus/kind-config.yaml diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 8ee7c8fa5..d4ded2439 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -368,7 +368,7 @@ parameters: ` if m.settings.ConnectionsEncrypted { // Encryption with kernel version <= 5.11 requires 'mounter: fuse'. For kernel version >= 5.12, it requires 'mounter: kernel'. - // Since the Github action Minikube has kernel version > 5.12, the setting is set to 'mounter: kernel'. + // Since the GitHub action runner has kernel version > 5.12, the setting is set to 'mounter: kernel'. sc += " mounter: kernel" } return sc diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index 44042dffb..0fc261a11 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -233,8 +233,12 @@ EOF # Set SA_CA_CRT to the PEM encoded CA cert used to talk to Kubernetes API SA_CA_CRT=$(kubectl -n "$ROOK_NAMESPACE" get secret "$VAULT_SA_SECRET_NAME" -o jsonpath="{.data['ca\.crt']}" | base64 --decode) - # get kubernetes endpoint - K8S_HOST=$(kubectl config view --minify --flatten -o jsonpath="{.clusters[0].cluster.server}") + # get kubernetes endpoint. Vault runs in-cluster, so it must reach the API server via the + # in-cluster service endpoint, not the kubeconfig's external server URL: under kind that URL is + # https://127.0.0.1:, which from inside the vault pod is its own localhost, so the + # TokenReview call fails. The kubernetes.default service is reachable from any pod, is covered by + # the API server cert SANs, and matches the issuer below. + K8S_HOST="https://kubernetes.default.svc.cluster.local:443" # enable kubernetes auth kubectl exec -ti vault-0 -- vault auth enable kubernetes diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index b935839c6..a9d1cd60f 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -306,6 +306,7 @@ function build_rook() { docker images if [[ "$build_type" == "build" ]]; then docker tag "$(docker images | awk '/build-/ {print $1}')" docker.io/rook/ceph:local-build + load_image_into_cluster docker.io/rook/ceph:local-build fi } @@ -313,6 +314,76 @@ function build_rook_all() { build_rook build.all } +function ensure_kind_is_available() { + # The kind-based CI helpers drive the cluster through the kind CLI; fail clearly if it is absent. + command -v kind >/dev/null 2>&1 || { + echo "the 'kind' command is required but was not found" >&2 + exit 1 + } +} + +function load_image_into_cluster() { + # Under kind, a locally built image lives only in the host docker daemon and must be + # explicitly imported into each cluster node's containerd before any pod can run it. + # + # Import through the node's own ctr rather than `kind load docker-image`: kind's loader parses + # the node's containerd config, and the kind version bundled with helm/kind-action cannot read + # the config version shipped by current kindest/node images ("unknown containerd config + # version: 4"). Piping `docker save` into the node's ctr sidesteps that version mismatch. + ensure_kind_is_available + local image="${1?image is required}" + local cluster="${KIND_CLUSTER_NAME:-kind}" + if ! kind get clusters 2>/dev/null | grep -qx "$cluster"; then + echo "load_image_into_cluster: no kind cluster '$cluster'" >&2 + exit 1 + fi + local node + for node in $(kind get nodes --name "$cluster"); do + docker save "$image" | docker exec -i "$node" ctr --namespace=k8s.io images import - + done +} + +function add_host_routes_to_cluster() { + # kind runs the cluster inside a docker network, so route the Service and pod CIDRs to the kind + # node so host-side tests (the `go test` process runs on the runner; e.g. an S3 PutObject to the + # RGW ClusterIP) can reach in-cluster services. + ensure_kind_is_available + local cluster="${KIND_CLUSTER_NAME:-kind}" + local node ip + node=$(kind get nodes --name "$cluster" | grep control-plane | head -1) + ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$node") + [ -n "$ip" ] || { echo "ERROR: no kind node IP for routing" >&2; return 1; } + # kind defaults: service subnet 10.96.0.0/16, pod subnet 10.244.0.0/16 + sudo ip route replace 10.96.0.0/16 via "$ip" + sudo ip route replace 10.244.0.0/16 via "$ip" + ip route show | grep -E '10\.(96|244)\.' || true # diagnostic print only +} + +function prepare_kind_node() { + # Prepare each kind node for the host-level operations rook performs against the node's host + # (remount /sys rw for krbd; install lvm2/cryptsetup for LVM/encryption OSDs). Best-effort: a + # failure must not break the raw-device OSD jobs that need none of it. + ensure_kind_is_available + local cluster="${KIND_CLUSTER_NAME:-kind}" + local node + for node in $(kind get nodes --name "$cluster"); do + # kind mounts the node's /sys read-only, so CSI's `rbd map --device-type krbd` fails with + # "rbd: sysfs write failed ... Read-only file system" writing /sys/bus/rbd/add. The node is + # privileged, so remount /sys read-write to let kernel RBD mapping work. + docker exec "$node" mount -o remount,rw /sys || + echo "WARNING: could not remount /sys rw in kind node $node" >&2 + # Rook provisions LVM- and encryption-backed OSDs by running lvm/cryptsetup in the node's + # mount namespace; kindest/node images don't ship those tools, so install them. + for _ in 1 2 3; do + if docker exec "$node" sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y lvm2 cryptsetup"; then + break + fi + echo "WARNING: could not install lvm2/cryptsetup into kind node $node; retrying" >&2 + sleep 5 + done + done +} + function validate_yaml() { cd "${REPO_DIR}/deploy/examples" kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml @@ -516,13 +587,18 @@ function wait_for_prepare_pod() { } function wait_for_cleanup_pod() { + # rook names the disk-cleanup job per node: cluster-cleanup-job-. Resolve the actual k8s + # node name rather than assuming it equals the runner hostname ($(uname -n)): under kind the + # node is "kind-control-plane", not the runner host. + local node + node="$(kubectl get nodes -o name | head -1 | cut -d/ -f2)" timeout 180 bash <