forked from rook/rook
Merge pull request #17903 from rook/mergify/bp/release-1.20/pr-17822
ci: convert integration and canary CI from minikube to kind (backport #17822)
This commit is contained in:
mergify[bot]
@@ -280,15 +280,15 @@ jobs:
|
||||
kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1c -nrook-ceph
|
||||
|
||||
# pass correct flags
|
||||
kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03
|
||||
kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values kind-control-plane,arbitrary,value
|
||||
# pass the pool which is not exists
|
||||
if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then
|
||||
if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values kind-control-plane,arbitrary,value); then
|
||||
echo "script run completed with stderr error after passing the wrong pools: $output"
|
||||
else
|
||||
echo "script failed because wrong pools doesn't exist"
|
||||
fi
|
||||
# dont pass all topology flags
|
||||
if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then
|
||||
if output=$(kubectl -n rook-ceph exec "$toolbox" -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values kind-control-plane,arbitrary,value); then
|
||||
echo "script run completed with stderr error after passing the wrong flags: $output"
|
||||
else
|
||||
echo "script failed because topology-failure-domain-label is missing"
|
||||
@@ -1831,7 +1831,7 @@ jobs:
|
||||
timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -gt 3 ]; do echo "waiting for connection to peer" && sleep 1; done'
|
||||
sockets=$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/)
|
||||
# shellcheck disable=SC2086 # word splitting of $sockets into separate socket paths is intentional
|
||||
status=$(for socket in $sockets; do minikube kubectl -- -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon "/var/run/ceph/$socket" help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done)
|
||||
status=$(for socket in $sockets; do kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon "/var/run/ceph/$socket" help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done)
|
||||
if [ "${#status}" -lt 1 ]; then echo "peer addition failed" && exit 1; fi
|
||||
|
||||
- name: display cephfilesystem and fs mirror daemon status
|
||||
|
||||
@@ -22,39 +22,27 @@ runs:
|
||||
with:
|
||||
go-version: "1.26"
|
||||
|
||||
- name: Install cri-dockerd
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
ARCH=$(go env GOARCH)
|
||||
# --fail so an HTTP error page is not saved as the .deb (dpkg then fails on the
|
||||
# corrupt archive), and retry transient download errors
|
||||
curl -fLO --retry 5 --retry-all-errors --retry-delay 10 https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.21/cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb
|
||||
sudo dpkg -i cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb
|
||||
rm -f cri-dockerd_0.3.21.3-0.ubuntu-focal_"${ARCH}".deb
|
||||
|
||||
- name: Setup Minikube
|
||||
uses: medyagh/setup-minikube@latest
|
||||
- name: create kind cluster
|
||||
uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
|
||||
with:
|
||||
# lock minikube version to ensure release branch CI doesn't fail when latest minikube no
|
||||
# longer supports older k8s version we lock to
|
||||
minikube-version: "1.38.0"
|
||||
kubernetes-version: ${{ inputs.kubernetes-version }}
|
||||
driver: none
|
||||
container-runtime: docker
|
||||
memory: 6g
|
||||
cpus: 2
|
||||
addons: ingress
|
||||
cni: calico
|
||||
# --force: when the requested kubernetes version is missing from minikube's compiled-in
|
||||
# version list (any version released after the pinned minikube), 'minikube start'
|
||||
# verifies the version with an anonymous GitHub API request, which is regularly
|
||||
# rate-limited on shared runner IPs and fails the job (K8S_FAIL_CONNECT, exit code 40).
|
||||
# The versions used here are pinned constants already validated by the PRs that bump
|
||||
# them, so that check adds nothing in CI; an invalid version would still fail fast at
|
||||
# the kubeadm/kubelet download.
|
||||
# --wait-timeout: minikube's default 6m node-ready wait expires on slow runners
|
||||
# (GUEST_START, exit code 80).
|
||||
start-args: --force --wait-timeout=15m
|
||||
# Pin kind; its release sets which kindest/node image versions are available.
|
||||
version: v0.32.0
|
||||
# Each matrix kubernetes-version needs a published kindest/node:<version> image.
|
||||
node_image: kindest/node:${{ inputs.kubernetes-version }}
|
||||
# See the config; it bind-mounts the host paths the disk-prep helpers need.
|
||||
config: tests/config/kind-config.yaml
|
||||
cluster_name: kind
|
||||
wait: 300s
|
||||
# Runner is ephemeral; some jobs hold node devices via the /dev mount, wedging teardown.
|
||||
ignore_failed_clean: "true"
|
||||
|
||||
- name: prepare the kind node
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: tests/scripts/github-action-helper.sh prepare_kind_node
|
||||
|
||||
- name: route host to cluster services
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: tests/scripts/github-action-helper.sh add_host_routes_to_cluster
|
||||
|
||||
- name: install deps
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
- name: Create KinD Cluster
|
||||
uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
|
||||
with:
|
||||
config: tests/scripts/multus/kind-config.yaml
|
||||
config: tests/config/multus/kind-config.yaml
|
||||
cluster_name: kind
|
||||
|
||||
- name: consider (pre-job) debugging
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# Kind cluster configuration for Rook's canary and integration test CI.
|
||||
#
|
||||
# Kind runs the node as a container, so host resources are not visible to it by default. The
|
||||
# extraMounts below bind-mount the resources the host-based disk-prep helpers (use_local_disk*,
|
||||
# create_partitions_for_osds, blockDevicePV.sh, localPathPV.sh, ...) depend on:
|
||||
# - /dev : block devices, partitions and LVM devices created on the host
|
||||
# appear inside the node (shared devtmpfs) and, in turn, inside the OSD
|
||||
# pods, which hostPath-mount the node's /dev.
|
||||
# - /var/lib/rook : dataDirHostPath and the local-PV mon directories created on the host.
|
||||
# - /run/udev : ceph-volume reads the udev database (/run/udev/data) to inventory a
|
||||
# disk; the kind node runs no udevd, so without the host's populated db
|
||||
# ceph-volume fails with "No udev data could be retrieved" and rook skips
|
||||
# the device. The runner host runs udevd, so bind-mount its db.
|
||||
# HostToContainer propagation ensures devices created on the host after the cluster is up
|
||||
# (e.g. partitions and extra iSCSI disks set up by per-job steps) also appear inside the node.
|
||||
kind: Cluster
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
nodes:
|
||||
- role: control-plane
|
||||
extraMounts:
|
||||
- hostPath: /dev
|
||||
containerPath: /dev
|
||||
propagation: HostToContainer
|
||||
- hostPath: /var/lib/rook
|
||||
containerPath: /var/lib/rook
|
||||
propagation: HostToContainer
|
||||
- hostPath: /run/udev
|
||||
containerPath: /run/udev
|
||||
propagation: HostToContainer
|
||||
@@ -368,7 +368,7 @@ parameters:
|
||||
`
|
||||
if m.settings.ConnectionsEncrypted {
|
||||
// Encryption with kernel version <= 5.11 requires 'mounter: fuse'. For kernel version >= 5.12, it requires 'mounter: kernel'.
|
||||
// Since the Github action Minikube has kernel version > 5.12, the setting is set to 'mounter: kernel'.
|
||||
// Since the GitHub action runner has kernel version > 5.12, the setting is set to 'mounter: kernel'.
|
||||
sc += " mounter: kernel"
|
||||
}
|
||||
return sc
|
||||
|
||||
@@ -233,8 +233,12 @@ EOF
|
||||
# Set SA_CA_CRT to the PEM encoded CA cert used to talk to Kubernetes API
|
||||
SA_CA_CRT=$(kubectl -n "$ROOK_NAMESPACE" get secret "$VAULT_SA_SECRET_NAME" -o jsonpath="{.data['ca\.crt']}" | base64 --decode)
|
||||
|
||||
# get kubernetes endpoint
|
||||
K8S_HOST=$(kubectl config view --minify --flatten -o jsonpath="{.clusters[0].cluster.server}")
|
||||
# get kubernetes endpoint. Vault runs in-cluster, so it must reach the API server via the
|
||||
# in-cluster service endpoint, not the kubeconfig's external server URL: under kind that URL is
|
||||
# https://127.0.0.1:<host-port>, which from inside the vault pod is its own localhost, so the
|
||||
# TokenReview call fails. The kubernetes.default service is reachable from any pod, is covered by
|
||||
# the API server cert SANs, and matches the issuer below.
|
||||
K8S_HOST="https://kubernetes.default.svc.cluster.local:443"
|
||||
|
||||
# enable kubernetes auth
|
||||
kubectl exec -ti vault-0 -- vault auth enable kubernetes
|
||||
|
||||
@@ -308,6 +308,7 @@ function build_rook() {
|
||||
docker images
|
||||
if [[ "$build_type" == "build" ]]; then
|
||||
docker tag "$(docker images | awk '/build-/ {print $1}')" docker.io/rook/ceph:local-build
|
||||
load_image_into_cluster docker.io/rook/ceph:local-build
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -315,6 +316,76 @@ function build_rook_all() {
|
||||
build_rook build.all
|
||||
}
|
||||
|
||||
function ensure_kind_is_available() {
|
||||
# The kind-based CI helpers drive the cluster through the kind CLI; fail clearly if it is absent.
|
||||
command -v kind >/dev/null 2>&1 || {
|
||||
echo "the 'kind' command is required but was not found" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
function load_image_into_cluster() {
|
||||
# Under kind, a locally built image lives only in the host docker daemon and must be
|
||||
# explicitly imported into each cluster node's containerd before any pod can run it.
|
||||
#
|
||||
# Import through the node's own ctr rather than `kind load docker-image`: kind's loader parses
|
||||
# the node's containerd config, and the kind version bundled with helm/kind-action cannot read
|
||||
# the config version shipped by current kindest/node images ("unknown containerd config
|
||||
# version: 4"). Piping `docker save` into the node's ctr sidesteps that version mismatch.
|
||||
ensure_kind_is_available
|
||||
local image="${1?image is required}"
|
||||
local cluster="${KIND_CLUSTER_NAME:-kind}"
|
||||
if ! kind get clusters 2>/dev/null | grep -qx "$cluster"; then
|
||||
echo "load_image_into_cluster: no kind cluster '$cluster'" >&2
|
||||
exit 1
|
||||
fi
|
||||
local node
|
||||
for node in $(kind get nodes --name "$cluster"); do
|
||||
docker save "$image" | docker exec -i "$node" ctr --namespace=k8s.io images import -
|
||||
done
|
||||
}
|
||||
|
||||
function add_host_routes_to_cluster() {
|
||||
# kind runs the cluster inside a docker network, so route the Service and pod CIDRs to the kind
|
||||
# node so host-side tests (the `go test` process runs on the runner; e.g. an S3 PutObject to the
|
||||
# RGW ClusterIP) can reach in-cluster services.
|
||||
ensure_kind_is_available
|
||||
local cluster="${KIND_CLUSTER_NAME:-kind}"
|
||||
local node ip
|
||||
node=$(kind get nodes --name "$cluster" | grep control-plane | head -1)
|
||||
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$node")
|
||||
[ -n "$ip" ] || { echo "ERROR: no kind node IP for routing" >&2; return 1; }
|
||||
# kind defaults: service subnet 10.96.0.0/16, pod subnet 10.244.0.0/16
|
||||
sudo ip route replace 10.96.0.0/16 via "$ip"
|
||||
sudo ip route replace 10.244.0.0/16 via "$ip"
|
||||
ip route show | grep -E '10\.(96|244)\.' || true # diagnostic print only
|
||||
}
|
||||
|
||||
function prepare_kind_node() {
|
||||
# Prepare each kind node for the host-level operations rook performs against the node's host
|
||||
# (remount /sys rw for krbd; install lvm2/cryptsetup for LVM/encryption OSDs). Best-effort: a
|
||||
# failure must not break the raw-device OSD jobs that need none of it.
|
||||
ensure_kind_is_available
|
||||
local cluster="${KIND_CLUSTER_NAME:-kind}"
|
||||
local node
|
||||
for node in $(kind get nodes --name "$cluster"); do
|
||||
# kind mounts the node's /sys read-only, so CSI's `rbd map --device-type krbd` fails with
|
||||
# "rbd: sysfs write failed ... Read-only file system" writing /sys/bus/rbd/add. The node is
|
||||
# privileged, so remount /sys read-write to let kernel RBD mapping work.
|
||||
docker exec "$node" mount -o remount,rw /sys ||
|
||||
echo "WARNING: could not remount /sys rw in kind node $node" >&2
|
||||
# Rook provisions LVM- and encryption-backed OSDs by running lvm/cryptsetup in the node's
|
||||
# mount namespace; kindest/node images don't ship those tools, so install them.
|
||||
for _ in 1 2 3; do
|
||||
if docker exec "$node" sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y lvm2 cryptsetup"; then
|
||||
break
|
||||
fi
|
||||
echo "WARNING: could not install lvm2/cryptsetup into kind node $node; retrying" >&2
|
||||
sleep 5
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
function validate_yaml() {
|
||||
cd "${REPO_DIR}/deploy/examples"
|
||||
kubectl create -f crds.yaml -f common.yaml -f csi-operator.yaml
|
||||
@@ -518,13 +589,18 @@ function wait_for_prepare_pod() {
|
||||
}
|
||||
|
||||
function wait_for_cleanup_pod() {
|
||||
# rook names the disk-cleanup job per node: cluster-cleanup-job-<node>. Resolve the actual k8s
|
||||
# node name rather than assuming it equals the runner hostname ($(uname -n)): under kind the
|
||||
# node is "kind-control-plane", not the runner host.
|
||||
local node
|
||||
node="$(kubectl get nodes -o name | head -1 | cut -d/ -f2)"
|
||||
timeout 180 bash <<EOF
|
||||
until kubectl --namespace rook-ceph logs job/cluster-cleanup-job-$(uname -n); do
|
||||
until kubectl --namespace rook-ceph logs job/cluster-cleanup-job-${node}; do
|
||||
echo "waiting for cleanup up pod to be present"
|
||||
sleep 1
|
||||
done
|
||||
EOF
|
||||
kubectl --namespace rook-ceph logs --follow job/cluster-cleanup-job-"$(uname -n)"
|
||||
kubectl --namespace rook-ceph logs --follow job/cluster-cleanup-job-"${node}"
|
||||
}
|
||||
|
||||
function wait_for_ceph_to_be_ready() {
|
||||
@@ -887,6 +963,7 @@ function create_helm_tag() {
|
||||
helm_tag="$(cat _output/version)"
|
||||
build_image="$(docker images | awk '/build-/ {print $1}')"
|
||||
docker tag "${build_image}" "rook/ceph:${helm_tag}"
|
||||
load_image_into_cluster "rook/ceph:${helm_tag}"
|
||||
}
|
||||
|
||||
function test_multus_connections() {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
####################################################################################################
|
||||
# this daemonset configures each minikube or KinD node to route to the multus public net
|
||||
# this daemonset configures each KinD node to route to the multus public net
|
||||
# it is not suitable for production
|
||||
####################################################################################################
|
||||
apiVersion: apps/v1
|
||||
|
||||
@@ -7,7 +7,7 @@ OUTFILE="cli-test.log"
|
||||
kubectl create namespace "$NAMESPACE"
|
||||
|
||||
# create the expected serviceaccount in the namespace
|
||||
# in Minikube/KinD, the tool needs no special permissions, so just the SA is fine
|
||||
# in KinD, the tool needs no special permissions, so just the SA is fine
|
||||
kubectl create serviceaccount rook-ceph-system --namespace "$NAMESPACE"
|
||||
|
||||
./rook --log-level DEBUG multus validation run \
|
||||
|
||||
@@ -8,7 +8,7 @@ OUTFILE="overlap-test.log"
|
||||
kubectl create namespace "$NAMESPACE"
|
||||
|
||||
# create the expected serviceaccount in the namespace
|
||||
# in Minikube/KinD, the tool needs no special permissions, so just the SA is fine
|
||||
# in KinD, the tool needs no special permissions, so just the SA is fine
|
||||
kubectl create serviceaccount rook-ceph-system --namespace "$NAMESPACE"
|
||||
|
||||
sed \
|
||||
|
||||
Reference in New Issue
Block a user