Merge pull request #17903 from rook/mergify/bp/release-1.20/pr-17822

ci: convert integration and canary CI from minikube to kind (backport #17822)
This commit is contained in:
mergify[bot] authored and GitHub committed 2026-07-07 16:35:31 +00:00
commit e16ae7d32e
11 files changed
+143 -45

No files matched your search

+6 -2
View File
@@ -233,8 +233,12 @@ EOF
# Set SA_CA_CRT to the PEM encoded CA cert used to talk to Kubernetes API
SA_CA_CRT=$(kubectl -n "$ROOK_NAMESPACE" get secret "$VAULT_SA_SECRET_NAME" -o jsonpath="{.data['ca\.crt']}" | base64 --decode)
# get kubernetes endpoint
K8S_HOST=$(kubectl config view --minify --flatten -o jsonpath="{.clusters[0].cluster.server}")
# get kubernetes endpoint. Vault runs in-cluster, so it must reach the API server via the
# in-cluster service endpoint, not the kubeconfig's external server URL: under kind that URL is
# https://127.0.0.1:<host-port>, which from inside the vault pod is its own localhost, so the
# TokenReview call fails. The kubernetes.default service is reachable from any pod, is covered by
# the API server cert SANs, and matches the issuer below.
K8S_HOST="https://kubernetes.default.svc.cluster.local:443"
# enable kubernetes auth
kubectl exec -ti vault-0 -- vault auth enable kubernetes
+79 -2
View File
@@ -308,6 +308,7 @@ function build_rook() {
docker images
if [[ "$build_type" == "build" ]]; then
docker tag "$(docker images | awk '/build-/ {print $1}')" docker.io/rook/ceph:local-build
load_image_into_cluster docker.io/rook/ceph:local-build
fi
}
@@ -315,6 +316,76 @@ function build_rook_all() {
build_rook build.all
}
function ensure_kind_is_available() {
# The kind-based CI helpers drive the cluster through the kind CLI; fail clearly if it is absent.
command -v kind >/dev/null 2>&1 || {
echo "the 'kind' command is required but was not found" >&2
exit 1
}
}
function load_image_into_cluster() {
# Under kind, a locally built image lives only in the host docker daemon and must be
# explicitly imported into each cluster node's containerd before any pod can run it.
#
# Import through the node's own ctr rather than `kind load docker-image`: kind's loader parses
# the node's containerd config, and the kind version bundled with helm/kind-action cannot read
# the config version shipped by current kindest/node images ("unknown containerd config
# version: 4"). Piping `docker save` into the node's ctr sidesteps that version mismatch.
ensure_kind_is_available
local image="${1?image is required}"
local cluster="${KIND_CLUSTER_NAME:-kind}"
if ! kind get clusters 2>/dev/null | grep -qx "$cluster"; then
echo "load_image_into_cluster: no kind cluster '$cluster'" >&2
exit 1
fi
local node
for node in $(kind get nodes --name "$cluster"); do
docker save "$image" | docker exec -i "$node" ctr --namespace=k8s.io images import -
done
}
function add_host_routes_to_cluster() {
# kind runs the cluster inside a docker network, so route the Service and pod CIDRs to the kind
# node so host-side tests (the `go test` process runs on the runner; e.g. an S3 PutObject to the
# RGW ClusterIP) can reach in-cluster services.
ensure_kind_is_available
local cluster="${KIND_CLUSTER_NAME:-kind}"
local node ip
node=$(kind get nodes --name "$cluster" | grep control-plane | head -1)
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$node")
[ -n "$ip" ] || { echo "ERROR: no kind node IP for routing" >&2; return 1; }
# kind defaults: service subnet 10.96.0.0/16, pod subnet 10.244.0.0/16
sudo ip route replace 10.96.0.0/16 via "$ip"
sudo ip route replace 10.244.0.0/16 via "$ip"
ip route show | grep -E '10\.(96|244)\.' || true # diagnostic print only
}
function prepare_kind_node() {
# Prepare each kind node for the host-level operations rook performs against the node's host
# (remount /sys rw for krbd; install lvm2/cryptsetup for LVM/encryption OSDs). Best-effort: a
# failure must not break the raw-device OSD jobs that need none of it.
ensure_kind_is_available
local cluster="${KIND_CLUSTER_NAME:-kind}"
local node
for node in $(kind get nodes --name "$cluster"); do
# kind mounts the node's /sys read-only, so CSI's `rbd map --device-type krbd` fails with
# "rbd: sysfs write failed ... Read-only file system" writing /sys/bus/rbd/add. The node is
# privileged, so remount /sys read-write to let kernel RBD mapping work.
docker exec "$node" mount -o remount,rw /sys ||
echo "WARNING: could not remount /sys rw in kind node $node" >&2
# Rook provisions LVM- and encryption-backed OSDs by running lvm/cryptsetup in the node's
# mount namespace; kindest/node images don't ship those tools, so install them.
for _ in 1 2 3; do
if docker exec "$node" sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y lvm2 cryptsetup"; then
break
fi
echo "WARNING: could not install lvm2/cryptsetup into kind node $node; retrying" >&2
sleep 5
done
done
}
function validate_yaml() {
cd "${REPO_DIR}/deploy/examples"
kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml
@@ -518,13 +589,18 @@ function wait_for_prepare_pod() {
}
function wait_for_cleanup_pod() {
# rook names the disk-cleanup job per node: cluster-cleanup-job-<node>. Resolve the actual k8s
# node name rather than assuming it equals the runner hostname ($(uname -n)): under kind the
# node is "kind-control-plane", not the runner host.
local node
node="$(kubectl get nodes -o name | head -1 | cut -d/ -f2)"
timeout 180 bash <<EOF
until kubectl --namespace rook-ceph logs job/cluster-cleanup-job-$(uname -n); do
until kubectl --namespace rook-ceph logs job/cluster-cleanup-job-${node}; do
echo "waiting for cleanup up pod to be present"
sleep 1
done
EOF
kubectl --namespace rook-ceph logs --follow job/cluster-cleanup-job-"$(uname -n)"
kubectl --namespace rook-ceph logs --follow job/cluster-cleanup-job-"${node}"
}
function wait_for_ceph_to_be_ready() {
@@ -887,6 +963,7 @@ function create_helm_tag() {
helm_tag="$(cat _output/version)"
build_image="$(docker images | awk '/build-/ {print $1}')"
docker tag "${build_image}" "rook/ceph:${helm_tag}"
load_image_into_cluster "rook/ceph:${helm_tag}"
}
function test_multus_connections() {
+1 -1
View File
@@ -1,5 +1,5 @@
####################################################################################################
# this daemonset configures each minikube or KinD node to route to the multus public net
# this daemonset configures each KinD node to route to the multus public net
# it is not suitable for production
####################################################################################################
apiVersion: apps/v1
-7
View File
@@ -1,7 +0,0 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker
+1 -1
View File
@@ -7,7 +7,7 @@ OUTFILE="cli-test.log"
kubectl create namespace "$NAMESPACE"
# create the expected serviceaccount in the namespace
# in Minikube/KinD, the tool needs no special permissions, so just the SA is fine
# in KinD, the tool needs no special permissions, so just the SA is fine
kubectl create serviceaccount rook-ceph-system --namespace "$NAMESPACE"
./rook --log-level DEBUG multus validation run \
@@ -8,7 +8,7 @@ OUTFILE="overlap-test.log"
kubectl create namespace "$NAMESPACE"
# create the expected serviceaccount in the namespace
# in Minikube/KinD, the tool needs no special permissions, so just the SA is fine
# in KinD, the tool needs no special permissions, so just the SA is fine
kubectl create serviceaccount rook-ceph-system --namespace "$NAMESPACE"
sed \