diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index c6805e048..ea7854c0e 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -280,15 +280,15 @@ jobs: kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1c -nrook-ceph # pass correct flags - kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03 + kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values kind-control-plane,arbitrary,value # pass the pool which is not exists - if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then + if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values kind-control-plane,arbitrary,value); then echo "script run completed with stderr error after passing the wrong pools: $output" else echo "script failed because wrong pools doesn't exist" fi # dont pass all topology flags - if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then + if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values kind-control-plane,arbitrary,value); then echo "script run completed with stderr error after passing the wrong flags: $output" else echo "script failed because topology-failure-domain-label is missing" @@ -1831,7 +1831,7 @@ jobs: timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -gt 3 ]; do echo "waiting for connection to peer" && sleep 1; done' sockets=$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/) # shellcheck disable=SC2086 # word splitting of $sockets into separate socket paths is intentional - status=$(for socket in $sockets; do minikube kubectl -- -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon "/var/run/ceph/$socket" help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done) + status=$(for socket in $sockets; do kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon "/var/run/ceph/$socket" help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done) if [ "${#status}" -lt 1 ]; then echo "peer addition failed" && exit 1; fi - name: display cephfilesystem and fs mirror daemon status diff --git a/.github/workflows/integration-test-setup-cluster-resources/action.yaml b/.github/workflows/integration-test-setup-cluster-resources/action.yaml index 9f38b6da2..2cff5beae 100644 --- a/.github/workflows/integration-test-setup-cluster-resources/action.yaml +++ b/.github/workflows/integration-test-setup-cluster-resources/action.yaml @@ -22,39 +22,27 @@ runs: with: go-version: "1.26" - - name: Install cri-dockerd - shell: bash --noprofile --norc -eo pipefail -x {0} - run: | - ARCH=$(go env GOARCH) - # --fail so an HTTP error page is not saved as the .deb (dpkg then fails on the - # corrupt archive), and retry transient download errors - curl -fLO --retry 5 --retry-all-errors --retry-delay 10 https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.21/cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb - sudo dpkg -i cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb - rm -f cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb - - - name: Setup Minikube - uses: medyagh/setup-minikube@latest + - name: create kind cluster + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 with: - # lock minikube version to ensure release branch CI doesn't fail when latest minikube no - # longer supports older k8s version we lock to - minikube-version: "1.38.0" - kubernetes-version: ${{ inputs.kubernetes-version }} - driver: none - container-runtime: docker - memory: 6g - cpus: 2 - addons: ingress - cni: calico - # --force: when the requested kubernetes version is missing from minikube's compiled-in - # version list (any version released after the pinned minikube), 'minikube start' - # verifies the version with an anonymous GitHub API request, which is regularly - # rate-limited on shared runner IPs and fails the job (K8S_FAIL_CONNECT, exit code 40). - # The versions used here are pinned constants already validated by the PRs that bump - # them, so that check adds nothing in CI; an invalid version would still fail fast at - # the kubeadm/kubelet download. - # --wait-timeout: minikube's default 6m node-ready wait expires on slow runners - # (GUEST_START, exit code 80). - start-args: --force --wait-timeout=15m + # Pin kind; its release sets which kindest/node image versions are available. + version: v0.32.0 + # Each matrix kubernetes-version needs a published kindest/node: image. + node_image: kindest/node:${{ inputs.kubernetes-version }} + # See the config; it bind-mounts the host paths the disk-prep helpers need. + config: tests/config/kind-config.yaml + cluster_name: kind + wait: 300s + # Runner is ephemeral; some jobs hold node devices via the /dev mount, wedging teardown. + ignore_failed_clean: "true" + + - name: prepare the kind node + shell: bash --noprofile --norc -eo pipefail -x {0} + run: tests/scripts/github-action-helper.sh prepare_kind_node + + - name: route host to cluster services + shell: bash --noprofile --norc -eo pipefail -x {0} + run: tests/scripts/github-action-helper.sh add_host_routes_to_cluster - name: install deps shell: bash --noprofile --norc -eo pipefail -x {0} diff --git a/.github/workflows/multus.yaml b/.github/workflows/multus.yaml index 7db5749a0..c35b89ecb 100644 --- a/.github/workflows/multus.yaml +++ b/.github/workflows/multus.yaml @@ -46,7 +46,7 @@ jobs: - name: Create KinD Cluster uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 with: - config: tests/scripts/multus/kind-config.yaml + config: tests/config/multus/kind-config.yaml cluster_name: kind - name: consider (pre-job) debugging diff --git a/tests/config/kind-config.yaml b/tests/config/kind-config.yaml new file mode 100644 index 000000000..3494c2227 --- /dev/null +++ b/tests/config/kind-config.yaml @@ -0,0 +1,29 @@ +# Kind cluster configuration for Rook's canary and integration test CI. +# +# Kind runs the node as a container, so host resources are not visible to it by default. The +# extraMounts below bind-mount the resources the host-based disk-prep helpers (use_local_disk*, +# create_partitions_for_osds, blockDevicePV.sh, localPathPV.sh, ...) depend on: +# - /dev : block devices, partitions and LVM devices created on the host +# appear inside the node (shared devtmpfs) and, in turn, inside the OSD +# pods, which hostPath-mount the node's /dev. +# - /var/lib/rook : dataDirHostPath and the local-PV mon directories created on the host. +# - /run/udev : ceph-volume reads the udev database (/run/udev/data) to inventory a +# disk; the kind node runs no udevd, so without the host's populated db +# ceph-volume fails with "No udev data could be retrieved" and rook skips +# the device. The runner host runs udevd, so bind-mount its db. +# HostToContainer propagation ensures devices created on the host after the cluster is up +# (e.g. partitions and extra iSCSI disks set up by per-job steps) also appear inside the node. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraMounts: + - hostPath: /dev + containerPath: /dev + propagation: HostToContainer + - hostPath: /var/lib/rook + containerPath: /var/lib/rook + propagation: HostToContainer + - hostPath: /run/udev + containerPath: /run/udev + propagation: HostToContainer diff --git a/tests/scripts/multus/kind-config.yaml b/tests/config/multus/kind-config.yaml similarity index 100% rename from tests/scripts/multus/kind-config.yaml rename to tests/config/multus/kind-config.yaml diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 8ee7c8fa5..d4ded2439 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -368,7 +368,7 @@ parameters: ` if m.settings.ConnectionsEncrypted { // Encryption with kernel version <= 5.11 requires 'mounter: fuse'. For kernel version >= 5.12, it requires 'mounter: kernel'. - // Since the Github action Minikube has kernel version > 5.12, the setting is set to 'mounter: kernel'. + // Since the GitHub action runner has kernel version > 5.12, the setting is set to 'mounter: kernel'. sc += " mounter: kernel" } return sc diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index 44042dffb..0fc261a11 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -233,8 +233,12 @@ EOF # Set SA_CA_CRT to the PEM encoded CA cert used to talk to Kubernetes API SA_CA_CRT=$(kubectl -n "$ROOK_NAMESPACE" get secret "$VAULT_SA_SECRET_NAME" -o jsonpath="{.data['ca\.crt']}" | base64 --decode) - # get kubernetes endpoint - K8S_HOST=$(kubectl config view --minify --flatten -o jsonpath="{.clusters[0].cluster.server}") + # get kubernetes endpoint. Vault runs in-cluster, so it must reach the API server via the + # in-cluster service endpoint, not the kubeconfig's external server URL: under kind that URL is + # https://127.0.0.1:, which from inside the vault pod is its own localhost, so the + # TokenReview call fails. The kubernetes.default service is reachable from any pod, is covered by + # the API server cert SANs, and matches the issuer below. + K8S_HOST="https://kubernetes.default.svc.cluster.local:443" # enable kubernetes auth kubectl exec -ti vault-0 -- vault auth enable kubernetes diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index d2f1d6a62..989ffd659 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -308,6 +308,7 @@ function build_rook() { docker images if [[ "$build_type" == "build" ]]; then docker tag "$(docker images | awk '/build-/ {print $1}')" docker.io/rook/ceph:local-build + load_image_into_cluster docker.io/rook/ceph:local-build fi } @@ -315,6 +316,76 @@ function build_rook_all() { build_rook build.all } +function ensure_kind_is_available() { + # The kind-based CI helpers drive the cluster through the kind CLI; fail clearly if it is absent. + command -v kind >/dev/null 2>&1 || { + echo "the 'kind' command is required but was not found" >&2 + exit 1 + } +} + +function load_image_into_cluster() { + # Under kind, a locally built image lives only in the host docker daemon and must be + # explicitly imported into each cluster node's containerd before any pod can run it. + # + # Import through the node's own ctr rather than `kind load docker-image`: kind's loader parses + # the node's containerd config, and the kind version bundled with helm/kind-action cannot read + # the config version shipped by current kindest/node images ("unknown containerd config + # version: 4"). Piping `docker save` into the node's ctr sidesteps that version mismatch. + ensure_kind_is_available + local image="${1?image is required}" + local cluster="${KIND_CLUSTER_NAME:-kind}" + if ! kind get clusters 2>/dev/null | grep -qx "$cluster"; then + echo "load_image_into_cluster: no kind cluster '$cluster'" >&2 + exit 1 + fi + local node + for node in $(kind get nodes --name "$cluster"); do + docker save "$image" | docker exec -i "$node" ctr --namespace=k8s.io images import - + done +} + +function add_host_routes_to_cluster() { + # kind runs the cluster inside a docker network, so route the Service and pod CIDRs to the kind + # node so host-side tests (the `go test` process runs on the runner; e.g. an S3 PutObject to the + # RGW ClusterIP) can reach in-cluster services. + ensure_kind_is_available + local cluster="${KIND_CLUSTER_NAME:-kind}" + local node ip + node=$(kind get nodes --name "$cluster" | grep control-plane | head -1) + ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$node") + [ -n "$ip" ] || { echo "ERROR: no kind node IP for routing" >&2; return 1; } + # kind defaults: service subnet 10.96.0.0/16, pod subnet 10.244.0.0/16 + sudo ip route replace 10.96.0.0/16 via "$ip" + sudo ip route replace 10.244.0.0/16 via "$ip" + ip route show | grep -E '10\.(96|244)\.' || true # diagnostic print only +} + +function prepare_kind_node() { + # Prepare each kind node for the host-level operations rook performs against the node's host + # (remount /sys rw for krbd; install lvm2/cryptsetup for LVM/encryption OSDs). Best-effort: a + # failure must not break the raw-device OSD jobs that need none of it. + ensure_kind_is_available + local cluster="${KIND_CLUSTER_NAME:-kind}" + local node + for node in $(kind get nodes --name "$cluster"); do + # kind mounts the node's /sys read-only, so CSI's `rbd map --device-type krbd` fails with + # "rbd: sysfs write failed ... Read-only file system" writing /sys/bus/rbd/add. The node is + # privileged, so remount /sys read-write to let kernel RBD mapping work. + docker exec "$node" mount -o remount,rw /sys || + echo "WARNING: could not remount /sys rw in kind node $node" >&2 + # Rook provisions LVM- and encryption-backed OSDs by running lvm/cryptsetup in the node's + # mount namespace; kindest/node images don't ship those tools, so install them. + for _ in 1 2 3; do + if docker exec "$node" sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y lvm2 cryptsetup"; then + break + fi + echo "WARNING: could not install lvm2/cryptsetup into kind node $node; retrying" >&2 + sleep 5 + done + done +} + function validate_yaml() { cd "${REPO_DIR}/deploy/examples" kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml @@ -518,13 +589,18 @@ function wait_for_prepare_pod() { } function wait_for_cleanup_pod() { + # rook names the disk-cleanup job per node: cluster-cleanup-job-. Resolve the actual k8s + # node name rather than assuming it equals the runner hostname ($(uname -n)): under kind the + # node is "kind-control-plane", not the runner host. + local node + node="$(kubectl get nodes -o name | head -1 | cut -d/ -f2)" timeout 180 bash <