diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..86fdf48 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,47 @@ +name: Validate and deploy CoreDNS scheduling +on: [push, pull_request] +env: + KUBECTL_VERSION: v1.36.3 + KUBERNETES_API: https://host.containers.internal:6443 + KUBERNETES_TLS_SERVER_NAME: 192.168.0.2 +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash test.sh + deploy: + if: gitea.ref == 'refs/heads/main' + needs: validate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install kubectl + run: | + curl -fsSL -o "${RUNNER_TEMP}/kubectl" "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + curl -fsSL -o "${RUNNER_TEMP}/kubectl.sha256" "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" + printf '%s %s\n' "$(cat "${RUNNER_TEMP}/kubectl.sha256")" "${RUNNER_TEMP}/kubectl" | sha256sum -c + chmod 0700 "${RUNNER_TEMP}/kubectl" + - name: Configure Kubernetes access + env: + KUBE_CONFIG_BASE64: ${{ secrets.KUBE_CONFIG_BASE64 }} + run: | + test -n "${KUBE_CONFIG_BASE64}" + printf '%s' "${KUBE_CONFIG_BASE64}" | base64 -d > "${RUNNER_TEMP}/kubeconfig" + chmod 0600 "${RUNNER_TEMP}/kubeconfig" + export KUBECONFIG="${RUNNER_TEMP}/kubeconfig" + "${RUNNER_TEMP}/kubectl" config set-cluster cluster --server="${KUBERNETES_API}" --tls-server-name="${KUBERNETES_TLS_SERVER_NAME}" + - name: Apply and verify + run: | + export KUBECONFIG="${RUNNER_TEMP}/kubeconfig" + "${RUNNER_TEMP}/kubectl" -n kube-system patch deployment coredns --type=strategic --patch-file coredns-scheduling-patch.yaml --dry-run=server + "${RUNNER_TEMP}/kubectl" apply --server-side --force-conflicts --dry-run=server -f coredns-pdb.yaml + "${RUNNER_TEMP}/kubectl" -n kube-system patch deployment coredns --type=strategic --patch-file coredns-scheduling-patch.yaml + "${RUNNER_TEMP}/kubectl" apply --server-side --force-conflicts -f coredns-pdb.yaml + for attempt in {1..60}; do + state=$("${RUNNER_TEMP}/kubectl" -n kube-system get deployment coredns -o jsonpath='{.metadata.generation}|{.status.observedGeneration}|{.spec.replicas}|{.status.updatedReplicas}|{.status.readyReplicas}|{.status.availableReplicas}') + IFS='|' read -r generation observed desired updated ready available <<< "$state" + [[ "$generation" == "$observed" && "$desired" == "$updated" && "$desired" == "$ready" && "$desired" == "$available" ]] && exit 0 + sleep 5 + done + exit 1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d06984 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.kubeconfig +*.secret +*secrets*.env diff --git a/README.md b/README.md index adc7f47..d8c9caf 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,26 @@ -# coredns-deployment +# CoreDNS scheduling +CoreDNS runs with two replicas on distinct ODROID H3 worker nodes. The required +node affinity prevents it from being scheduled on the experimental control-plane +node (`arschrock`), while required pod anti-affinity prevents both replicas from +sharing a worker. + +Apply and verify the configuration with: + +```sh +kubectl -n kube-system patch deployment coredns \ + --type=strategic \ + --patch-file coredns-scheduling-patch.yaml +kubectl apply -f coredns-pdb.yaml +kubectl -n kube-system rollout status deployment/coredns +kubectl -n kube-system get pods -l k8s-app=kube-dns -o wide +``` + +The ODROID worker nodes must retain the label `odroid-h3=enabled`. + +`./install.sh` performs those steps for an administrator. Gitea Actions checks +the manifests on every branch and updates only the exact existing +`kube-system/coredns` Deployment and PodDisruptionBudget on `main`. Its identity +cannot read Secrets, create or delete resources, or change any other workload. +Run `./create-ci-kubeconfig.sh` once and store only its final line as the +repository Actions secret `KUBE_CONFIG_BASE64`. diff --git a/ci-deployer.yaml b/ci-deployer.yaml new file mode 100644 index 0000000..f08b7e0 --- /dev/null +++ b/ci-deployer.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns-config-deployer + namespace: kube-system +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: coredns-config-deployer + namespace: kube-system +rules: + - apiGroups: [apps] + resources: [deployments] + resourceNames: [coredns] + verbs: [get, patch, update] + - apiGroups: [policy] + resources: [poddisruptionbudgets] + resourceNames: [coredns] + verbs: [get, patch, update] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: coredns-config-deployer + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coredns-config-deployer +subjects: + - kind: ServiceAccount + name: coredns-config-deployer + namespace: kube-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: coredns-config-deployer-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: coredns-config-deployer +type: kubernetes.io/service-account-token diff --git a/coredns-pdb.yaml b/coredns-pdb.yaml new file mode 100644 index 0000000..ee1521c --- /dev/null +++ b/coredns-pdb.yaml @@ -0,0 +1,10 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: coredns + namespace: kube-system +spec: + minAvailable: 1 + selector: + matchLabels: + k8s-app: kube-dns diff --git a/coredns-scheduling-patch.yaml b/coredns-scheduling-patch.yaml new file mode 100644 index 0000000..a2408e2 --- /dev/null +++ b/coredns-scheduling-patch.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system +spec: + replicas: 2 + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: odroid-h3 + operator: In + values: + - enabled + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: kube-dns + topologyKey: kubernetes.io/hostname diff --git a/create-ci-kubeconfig.sh b/create-ci-kubeconfig.sh new file mode 100755 index 0000000..88d9afc --- /dev/null +++ b/create-ci-kubeconfig.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +namespace=kube-system +service_account=coredns-config-deployer +secret=coredns-config-deployer-token +ssh_host=${KUBECTL_SSH_HOST:-} +ssh_config=${SSH_CONFIG:-} +cluster_kubectl() { if [[ -n "$ssh_host" ]]; then if [[ -n "$ssh_config" ]]; then ssh -F "$ssh_config" "$ssh_host" kubectl "$@"; else ssh "$ssh_host" kubectl "$@"; fi; else kubectl "$@"; fi; } +if [[ -n "$ssh_host" ]]; then cluster_kubectl apply -f - < "$project_dir/ci-deployer.yaml" >&2; else cluster_kubectl apply -f "$project_dir/ci-deployer.yaml" >&2; fi +for attempt in {1..30}; do + if cluster_kubectl -n "$namespace" get secret "$secret" -o jsonpath='{.data.token}' 2>/dev/null | grep -q .; then break; fi + [[ "$attempt" != 30 ]] || { echo 'Timed out waiting for service-account token' >&2; exit 1; } + sleep 1 +done +work=$(mktemp -d); trap 'rm -rf "$work"' EXIT +server=$(cluster_kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}') +cluster_kubectl -n "$namespace" get secret "$secret" -o jsonpath='{.data.ca\.crt}' | base64 -d > "$work/ca.crt" +token=$(cluster_kubectl -n "$namespace" get secret "$secret" -o jsonpath='{.data.token}' | base64 -d) +export KUBECONFIG="$work/config" +kubectl config set-cluster cluster --server="$server" --certificate-authority="$work/ca.crt" --embed-certs=true >/dev/null +kubectl config set-credentials "$service_account" --token="$token" >/dev/null +kubectl config set-context coredns --cluster=cluster --user="$service_account" --namespace="$namespace" >/dev/null +kubectl config use-context coredns >/dev/null +echo 'Store this line as Gitea Actions secret KUBE_CONFIG_BASE64; do not commit it.' >&2 +base64 -w0 "$KUBECONFIG"; printf '\n' diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..3646f15 --- /dev/null +++ b/install.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +project_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +kubectl -n kube-system patch deployment coredns --type=strategic \ + --patch-file "$project_dir/coredns-scheduling-patch.yaml" +kubectl apply -f "$project_dir/coredns-pdb.yaml" +kubectl -n kube-system rollout status deployment/coredns --timeout=5m diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..bbdf793 --- /dev/null +++ b/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +sh -n "$dir/install.sh" +bash -n "$dir/create-ci-kubeconfig.sh" "$dir/test.sh" +grep -Fq 'namespace: kube-system' "$dir/coredns-scheduling-patch.yaml" +grep -Fq 'odroid-h3' "$dir/coredns-scheduling-patch.yaml" +grep -Fq 'requiredDuringSchedulingIgnoredDuringExecution' "$dir/coredns-scheduling-patch.yaml" +image=ghcr.io/yannh/kubeconform:v0.7.0@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c +docker run --rm -i "$image" -strict -exit-on-error -summary < "$dir/coredns-pdb.yaml" +docker run --rm -i "$image" -strict -exit-on-error -summary < "$dir/ci-deployer.yaml" +echo 'CoreDNS scheduling manifests and least-privilege CI resources are valid.'