From 05495786f5eca25d0336292fd80d9ebfdeefa300 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Tue, 20 Oct 2026 16:42:07 +0200 Subject: [PATCH] Retire direct Elastic Kubernetes deployment after Argo adoption --- .gitea/workflows/deploy.yml | 116 -------------------------------- .gitea/workflows/validate.yml | 11 ++++ README.md | 63 ++++++------------ ci-deployer.yaml | 108 ------------------------------ create-ci-kubeconfig.sh | 66 ------------------- install.sh | 121 ---------------------------------- uninstall.sh | 23 ------- 7 files changed, 31 insertions(+), 477 deletions(-) delete mode 100644 .gitea/workflows/deploy.yml create mode 100644 .gitea/workflows/validate.yml delete mode 100644 ci-deployer.yaml delete mode 100755 create-ci-kubeconfig.sh delete mode 100755 install.sh delete mode 100755 uninstall.sh diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml deleted file mode 100644 index 0d5149b..0000000 --- a/.gitea/workflows/deploy.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Validate and deploy Elastic Stack - -on: - push: - pull_request: - -env: - KUBECONFORM_VERSION: v0.7.0 - KUBECTL_VERSION: v1.36.3 - KUBERNETES_API: https://host.containers.internal:6443 - KUBERNETES_TLS_SERVER_NAME: 192.168.0.2 - NAMESPACE: elastic - -jobs: - validate: - name: Validate manifests - runs-on: ubuntu-latest - steps: - - name: Check out source - uses: actions/checkout@v4 - - - name: Validate scripts and Kubernetes resources - run: | - bash -n install.sh install-traefik-assets.sh check.sh configure-router.sh uninstall.sh create-ci-kubeconfig.sh - jq --exit-status . traefik-access-ilm.json traefik-access-pipeline.json traefik-access-component-template.json traefik-traffic-dashboard.json >/dev/null - docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ - -strict -summary -ignore-missing-schemas < stack.yaml - docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ - -strict -summary -ignore-missing-schemas < netflow.yaml - docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ - -strict -summary -ignore-missing-schemas < ingress.yaml - docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ - -strict -summary -ignore-missing-schemas < traefik-access.yaml - docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ - -strict -summary < ci-deployer.yaml - - deploy: - name: Deploy to Kubernetes - if: gitea.ref == 'refs/heads/main' - needs: validate - runs-on: ubuntu-latest - steps: - - name: Check out source - uses: actions/checkout@v4 - - - name: Install kubectl - run: | - curl --fail --silent --show-error --location \ - --output "${RUNNER_TEMP}/kubectl" \ - "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - chmod 0700 "${RUNNER_TEMP}/kubectl" - - - name: Configure Kubernetes access - env: - KUBE_CONFIG_BASE64: ${{ secrets.KUBE_CONFIG_BASE64 }} - run: | - test -n "${KUBE_CONFIG_BASE64}" - printf '%s' "${KUBE_CONFIG_BASE64}" | base64 --decode > "${RUNNER_TEMP}/kubeconfig" - chmod 0600 "${RUNNER_TEMP}/kubeconfig" - export KUBECONFIG="${RUNNER_TEMP}/kubeconfig" - "${RUNNER_TEMP}/kubectl" config set-cluster cluster \ - --server="${KUBERNETES_API}" \ - --tls-server-name="${KUBERNETES_TLS_SERVER_NAME}" - - - name: Apply and verify - run: | - export KUBECONFIG="${RUNNER_TEMP}/kubeconfig" - "${RUNNER_TEMP}/kubectl" apply --dry-run=server --validate=false \ - --filename stack.yaml \ - --filename netflow.yaml \ - --filename ingress.yaml \ - --filename traefik-access.yaml - "${RUNNER_TEMP}/kubectl" apply --validate=false \ - --filename stack.yaml \ - --filename netflow.yaml \ - --filename ingress.yaml \ - --filename traefik-access.yaml - - for attempt in {1..240}; do - es_health=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get elasticsearch/elasticsearch --output=jsonpath='{.status.health}') - es_nodes=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get elasticsearch/elasticsearch --output=jsonpath='{.status.availableNodes}') - kb_health=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get kibana/kibana --output=jsonpath='{.status.health}') - kb_nodes=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get kibana/kibana --output=jsonpath='{.status.availableNodes}') - beat_health=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get beat/netflow --output=jsonpath='{.status.health}') - beat_nodes=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get beat/netflow --output=jsonpath='{.status.availableNodes}') - agent_health=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get agent/traefik-access --output=jsonpath='{.status.health}') - agent_nodes=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get agent/traefik-access --output=jsonpath='{.status.availableNodes}') - agent_expected=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get agent/traefik-access --output=jsonpath='{.status.expectedNodes}') - load_balancer_ip=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get service/netflow --output=jsonpath='{.status.loadBalancer.ingress[0].ip}') - - echo "Check ${attempt}/240: Elasticsearch ${es_health:-unknown} (${es_nodes:-0}/3), Kibana ${kb_health:-unknown} (${kb_nodes:-0}/2), Filebeat ${beat_health:-unknown} (${beat_nodes:-0}/1), Agent ${agent_health:-unknown} (${agent_nodes:-0}/${agent_expected:-0}), LB=${load_balancer_ip:-pending}" - if [[ "${es_health}" == "green" \ - && "${es_nodes}" == "3" \ - && "${kb_health}" == "green" \ - && "${kb_nodes}" == "2" \ - && "${beat_health}" == "green" \ - && "${beat_nodes}" == "1" \ - && "${agent_health}" == "green" \ - && -n "${agent_expected}" \ - && "${agent_expected}" != "0" \ - && "${agent_nodes}" == "${agent_expected}" \ - && "${load_balancer_ip}" == "192.168.0.20" ]]; then - echo "Elastic Stack successfully rolled out" - exit 0 - fi - - sleep 5 - done - - echo "Elastic Stack did not become ready within twenty minutes" >&2 - exit 1 diff --git a/.gitea/workflows/validate.yml b/.gitea/workflows/validate.yml new file mode 100644 index 0000000..f62057c --- /dev/null +++ b/.gitea/workflows/validate.yml @@ -0,0 +1,11 @@ +name: Validate Elastic asset tooling +on: [push, pull_request] +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Validate scripts and Kibana assets + run: | + bash -n install-traefik-assets.sh check.sh configure-router.sh + jq --exit-status . traefik-access-ilm.json traefik-access-pipeline.json traefik-access-component-template.json traefik-traffic-dashboard.json >/dev/null diff --git a/README.md b/README.md index 19b66e8..345a91e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Elastic network-observability stack +The live Kubernetes operator and ECK resources are now owned by Argo CD in +[homelab-gitops](https://code.brunner.ninja/feedc0de/homelab-gitops/src/branch/main/apps/elastic). +The Kubernetes manifests in this checkout are historical reference only; +editing or pushing them does not deploy them. This repository remains active +for the MikroTik router configuration and Kibana pipeline/dashboard assets. + Elasticsearch, Kibana, a dedicated MikroTik IPFIX collector, and a Traefik access-log collector managed by Elastic Cloud on Kubernetes (ECK). Everything runs in the `elastic` namespace. @@ -26,16 +32,16 @@ Traefik JSON access logs --> Elastic Agent on each node Elasticsearch ``` -- ECK operator `3.5.0`, restricted to managing the `elastic` namespace. -- Elasticsearch `9.5.2`: three nodes, 12 GiB memory each, and three expandable +- ECK operator, restricted to managing the `elastic` namespace. +- Elasticsearch: three nodes, 12 GiB memory each, and three expandable 60 GiB `rook-ceph-block-ec` volumes (180 GiB / about 193 GB total, including Elasticsearch's own shard replicas). The RBD image metadata remains on the replicated pool, while bulk data uses the 3+2 erasure-coded pool. -- Kibana `9.5.2`: two 4 GiB replicas at +- Kibana: two 4 GiB replicas at . -- Filebeat `9.5.2`: one stateful UDP collector. Keeping a single collector +- Filebeat: one stateful UDP collector. Keeping a single collector ensures all IPFIX templates and their flow packets reach the same process. -- Elastic Agent `9.5.2`: one pod per Traefik node, reading container access +- Elastic Agent: one pod per Traefik node, reading container access logs into the `logs-traefik.access-default` data stream. - All Stack components use Elastic's hardened Wolfi images. The standard UBI 10 images require the x86-64-v3 CPU baseline, which the Odroid H3 CPUs do @@ -50,22 +56,21 @@ Traefik JSON access logs --> Elastic Agent on each node retain their operator defaults. PVC capacity is a hard 180 GiB aggregate ceiling, but Elasticsearch disk-watermark safeguards will intervene earlier. -## Install +## Router and Kibana assets From a host with cluster-admin `kubectl` and Helm access: ```sh -./install.sh ./configure-router.sh +./install-traefik-assets.sh ./check.sh ``` -From the current remote location, copy the directory to `dahoam`, deploy from -there, and configure the router through the SSH jump host: +To run the external asset tooling from `dahoam`, copy the directory there and +configure the router through the SSH jump host: ```sh rsync -av elastic/ dahoam:elastic/ -ssh dahoam 'cd elastic && ./install.sh' JUMP_HOST=dahoam \ ROUTER_HOST=admin@192.168.0.254 \ ROUTER_HOST_KEY_ALIAS=newgw \ @@ -119,37 +124,9 @@ not logged because they frequently contain tokens. Run `./check.sh` to inspect readiness, recent collector logs, and the number of stored `netflow.log` events. -## Gitea Actions CD +## Gitea Actions -The ECK operator and initial resources must be bootstrapped once with -`install.sh`. CI subsequently updates only the committed stack resources; it -cannot read Secrets, modify RBAC, or change the operator. - -Create the restricted deployer and generate its kubeconfig: - -```sh -KUBECTL_SSH_HOST=dahoam \ -SSH_CONFIG="$HOME/.ssh/config" \ -./create-ci-kubeconfig.sh -``` - -Create one repository Actions secret: - -- `KUBE_CONFIG_BASE64`: the single base64 line printed by the command above. - -No Elasticsearch, Kibana, router, DNS, or certificate secret belongs in -Gitea. ECK and cert-manager create and rotate the live credentials inside the -cluster. - -## Removal - -This destroys the indices, Kibana saved objects, credentials, and all three -PVCs: - -```sh -./uninstall.sh --yes -``` - -The script removes the Helm release and namespace but intentionally retains -the cluster-scoped ECK CRDs. Removing CRDs is unsafe if another ECK deployment -is ever added to the cluster. +CI validates the retained router and Kibana asset tooling only. It has no +Kubernetes credentials and cannot deploy the ECK stack. Argo CD owns its +Kubernetes desired state; do not manually uninstall its Elasticsearch custom +resource because its claim deletion policy can remove all three data PVCs. diff --git a/ci-deployer.yaml b/ci-deployer.yaml deleted file mode 100644 index 0df6a8e..0000000 --- a/ci-deployer.yaml +++ /dev/null @@ -1,108 +0,0 @@ -# Bootstrap once with an administrator identity. CI can update only the -# existing Elastic resources and cannot read Secrets or change RBAC. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: elastic-deployer - namespace: elastic -automountServiceAccountToken: false ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: elastic-deployer - namespace: elastic -rules: - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - - netflow-ilm-policy - verbs: - - get - - patch - - update - - apiGroups: - - "" - resources: - - services - resourceNames: - - netflow - verbs: - - get - - patch - - update - - apiGroups: - - elasticsearch.k8s.elastic.co - resources: - - elasticsearches - resourceNames: - - elasticsearch - verbs: - - get - - patch - - update - - apiGroups: - - kibana.k8s.elastic.co - resources: - - kibanas - resourceNames: - - kibana - verbs: - - get - - patch - - update - - apiGroups: - - beat.k8s.elastic.co - resources: - - beats - resourceNames: - - netflow - verbs: - - get - - patch - - update - - apiGroups: - - agent.k8s.elastic.co - resources: - - agents - resourceNames: - - traefik-access - verbs: - - get - - patch - - update - - apiGroups: - - networking.k8s.io - resources: - - ingresses - resourceNames: - - kibana - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: elastic-deployer - namespace: elastic -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: elastic-deployer -subjects: - - kind: ServiceAccount - name: elastic-deployer - namespace: elastic ---- -apiVersion: v1 -kind: Secret -metadata: - name: elastic-deployer-token - namespace: elastic - annotations: - kubernetes.io/service-account.name: elastic-deployer -type: kubernetes.io/service-account-token diff --git a/create-ci-kubeconfig.sh b/create-ci-kubeconfig.sh deleted file mode 100755 index e49e9c0..0000000 --- a/create-ci-kubeconfig.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -namespace=${NAMESPACE:-elastic} -service_account=${SERVICE_ACCOUNT:-elastic-deployer} -secret=${TOKEN_SECRET:-elastic-deployer-token} -ssh_host=${KUBECTL_SSH_HOST:-} -ssh_config=${SSH_CONFIG:-} - -cluster_kubectl() { - if [[ -n "${ssh_host}" ]]; then - if [[ -n "${ssh_config}" ]]; then - ssh -F "${ssh_config}" "${ssh_host}" kubectl "$@" - else - ssh "${ssh_host}" kubectl "$@" - fi - else - kubectl "$@" - fi -} - -if [[ -n "${ssh_host}" ]]; then - cluster_kubectl apply --filename=- < "${project_dir}/ci-deployer.yaml" >&2 -else - cluster_kubectl apply --filename "${project_dir}/ci-deployer.yaml" >&2 -fi - -for attempt in {1..30}; do - if cluster_kubectl --namespace "${namespace}" get secret "${secret}" \ - --output=jsonpath='{.data.token}' 2>/dev/null | grep -q .; then - break - fi - if [[ "${attempt}" == 30 ]]; then - echo "Timed out waiting for the service-account token" >&2 - exit 1 - fi - sleep 1 -done - -workdir=$(mktemp --directory) -trap 'rm -rf "${workdir}"' EXIT - -server=$(cluster_kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}') -cluster_kubectl --namespace "${namespace}" get secret "${secret}" \ - --output=jsonpath='{.data.ca\.crt}' | base64 --decode > "${workdir}/ca.crt" -token=$(cluster_kubectl --namespace "${namespace}" get secret "${secret}" \ - --output=jsonpath='{.data.token}' | base64 --decode) - -export KUBECONFIG="${workdir}/config" -kubectl config set-cluster cluster \ - --server="${server}" \ - --certificate-authority="${workdir}/ca.crt" \ - --embed-certs=true >/dev/null -kubectl config set-credentials "${service_account}" --token="${token}" >/dev/null -kubectl config set-context elastic \ - --cluster=cluster \ - --user="${service_account}" \ - --namespace="${namespace}" >/dev/null -kubectl config use-context elastic >/dev/null - -echo "Store the following single line as the Gitea Actions secret KUBE_CONFIG_BASE64." >&2 -echo "Treat it as a password; do not commit it." >&2 -base64 --wrap=0 "${KUBECONFIG}" -printf '\n' diff --git a/install.sh b/install.sh deleted file mode 100755 index a16d705..0000000 --- a/install.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -namespace=elastic -operator_version=3.5.0 - -for command in helm kubectl; do - if ! command -v "${command}" >/dev/null 2>&1; then - echo "Required command not found: ${command}" >&2 - exit 1 - fi -done - -for dependency in storageclass/rook-ceph-block-ec; do - if ! kubectl get "${dependency}" >/dev/null 2>&1; then - echo "Required Kubernetes object not found: ${dependency}" >&2 - exit 1 - fi -done - -helm repo add elastic https://helm.elastic.co --force-update -helm repo update elastic - -helm upgrade --install elastic-operator elastic/eck-operator \ - --namespace "${namespace}" \ - --create-namespace \ - --version "${operator_version}" \ - --values "${project_dir}/operator-values.yaml" \ - --wait \ - --timeout 10m - -for crd in \ - elasticsearches.elasticsearch.k8s.elastic.co \ - kibanas.kibana.k8s.elastic.co \ - beats.beat.k8s.elastic.co \ - agents.agent.k8s.elastic.co; do - kubectl wait --for=condition=Established "crd/${crd}" --timeout=2m -done - -kubectl apply --dry-run=server --validate=false \ - --filename "${project_dir}/stack.yaml" \ - --filename "${project_dir}/netflow.yaml" \ - --filename "${project_dir}/ingress.yaml" -kubectl apply --validate=false \ - --filename "${project_dir}/stack.yaml" \ - --filename "${project_dir}/netflow.yaml" \ - --filename "${project_dir}/ingress.yaml" - -for attempt in {1..360}; do - es_health=$(kubectl --namespace "${namespace}" get elasticsearch/elasticsearch \ - --output=jsonpath='{.status.health}') - es_nodes=$(kubectl --namespace "${namespace}" get elasticsearch/elasticsearch \ - --output=jsonpath='{.status.availableNodes}') - kb_health=$(kubectl --namespace "${namespace}" get kibana/kibana \ - --output=jsonpath='{.status.health}') - kb_nodes=$(kubectl --namespace "${namespace}" get kibana/kibana \ - --output=jsonpath='{.status.availableNodes}') - beat_health=$(kubectl --namespace "${namespace}" get beat/netflow \ - --output=jsonpath='{.status.health}') - beat_nodes=$(kubectl --namespace "${namespace}" get beat/netflow \ - --output=jsonpath='{.status.availableNodes}') - load_balancer_ip=$(kubectl --namespace "${namespace}" get service/netflow \ - --output=jsonpath='{.status.loadBalancer.ingress[0].ip}') - - echo "Check ${attempt}/360: Elasticsearch ${es_health:-unknown} (${es_nodes:-0}/3), Kibana ${kb_health:-unknown} (${kb_nodes:-0}/2), Filebeat ${beat_health:-unknown} (${beat_nodes:-0}/1), LB=${load_balancer_ip:-pending}" - if [[ "${es_health}" == "green" \ - && "${es_nodes}" == "3" \ - && "${kb_health}" == "green" \ - && "${kb_nodes}" == "2" \ - && "${beat_health}" == "green" \ - && "${beat_nodes}" == "1" \ - && "${load_balancer_ip}" == "192.168.0.20" ]]; then - break - fi - - if [[ "${attempt}" == "360" ]]; then - echo "Timed out waiting for the complete Elastic Stack rollout" >&2 - exit 1 - fi - sleep 5 -done - -"${project_dir}/install-traefik-assets.sh" - -kubectl apply --dry-run=server --validate=false \ - --filename "${project_dir}/traefik-access.yaml" -kubectl apply --validate=false \ - --filename "${project_dir}/traefik-access.yaml" - -for attempt in {1..240}; do - agent_health=$(kubectl --namespace "${namespace}" get agent/traefik-access \ - --output=jsonpath='{.status.health}') - agent_nodes=$(kubectl --namespace "${namespace}" get agent/traefik-access \ - --output=jsonpath='{.status.availableNodes}') - agent_expected=$(kubectl --namespace "${namespace}" get agent/traefik-access \ - --output=jsonpath='{.status.expectedNodes}') - - echo "Agent check ${attempt}/240: ${agent_health:-unknown} (${agent_nodes:-0}/${agent_expected:-0})" - if [[ "${agent_health}" == "green" \ - && -n "${agent_expected}" \ - && "${agent_expected}" != "0" \ - && "${agent_nodes}" == "${agent_expected}" ]]; then - break - fi - - if [[ "${attempt}" == "240" ]]; then - echo "Timed out waiting for the Traefik access-log agents" >&2 - exit 1 - fi - sleep 5 -done - -echo -echo "Elastic Stack is ready in namespace ${namespace}." -echo "Kibana: https://kibana.brunner.ninja" -echo "NetFlow collector: ${load_balancer_ip}:2055/UDP" -echo "Traefik access logs: logs-traefik.access-default" -echo "Retrieve the Kibana 'elastic' password with:" -echo "kubectl get secret elasticsearch-es-elastic-user -n ${namespace} -o go-template='{{.data.elastic | base64decode}}{{\"\\n\"}}'" diff --git a/uninstall.sh b/uninstall.sh deleted file mode 100755 index 6f96dd9..0000000 --- a/uninstall.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -namespace=elastic - -if [[ "${1:-}" != "--yes" ]]; then - echo "This permanently deletes namespace ${namespace}, all Elasticsearch data, PVCs, credentials, and Kibana objects." >&2 - echo "Run $0 --yes to continue." >&2 - exit 1 -fi - -kubectl delete --ignore-not-found \ - --filename "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/ingress.yaml" \ - --filename "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/traefik-access.yaml" \ - --filename "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/netflow.yaml" \ - --filename "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/stack.yaml" - -helm uninstall elastic-operator --namespace "${namespace}" --wait || true -kubectl delete namespace "${namespace}" --ignore-not-found --wait=true - -echo "Deleted the Elastic Stack and namespace ${namespace}." -echo "ECK CRDs are intentionally retained because they are cluster-scoped."