Retire direct Elastic Kubernetes deployment after Argo adoption

.gitea/workflows/deploy.yml

@@ -1,116 +0,0 @@
-name: Validate and deploy Elastic Stack
-
-on:
-  push:
-  pull_request:
-
-env:
-  KUBECONFORM_VERSION: v0.7.0
-  KUBECTL_VERSION: v1.36.3
-  KUBERNETES_API: https://host.containers.internal:6443
-  KUBERNETES_TLS_SERVER_NAME: 192.168.0.2
-  NAMESPACE: elastic
-
-jobs:
-  validate:
-    name: Validate manifests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out source
-        uses: actions/checkout@v4
-
-      - name: Validate scripts and Kubernetes resources
-        run: |
-          bash -n install.sh install-traefik-assets.sh check.sh configure-router.sh uninstall.sh create-ci-kubeconfig.sh
-          jq --exit-status . traefik-access-ilm.json traefik-access-pipeline.json traefik-access-component-template.json traefik-traffic-dashboard.json >/dev/null
-          docker run --rm --interactive \
-            "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \
-            -strict -summary -ignore-missing-schemas < stack.yaml
-          docker run --rm --interactive \
-            "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \
-            -strict -summary -ignore-missing-schemas < netflow.yaml
-          docker run --rm --interactive \
-            "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \
-            -strict -summary -ignore-missing-schemas < ingress.yaml
-          docker run --rm --interactive \
-            "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \
-            -strict -summary -ignore-missing-schemas < traefik-access.yaml
-          docker run --rm --interactive \
-            "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \
-            -strict -summary < ci-deployer.yaml
-
-  deploy:
-    name: Deploy to Kubernetes
-    if: gitea.ref == 'refs/heads/main'
-    needs: validate
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out source
-        uses: actions/checkout@v4
-
-      - name: Install kubectl
-        run: |
-          curl --fail --silent --show-error --location \
-            --output "${RUNNER_TEMP}/kubectl" \
-            "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
-          chmod 0700 "${RUNNER_TEMP}/kubectl"
-
-      - name: Configure Kubernetes access
-        env:
-          KUBE_CONFIG_BASE64: ${{ secrets.KUBE_CONFIG_BASE64 }}
-        run: |
-          test -n "${KUBE_CONFIG_BASE64}"
-          printf '%s' "${KUBE_CONFIG_BASE64}" | base64 --decode > "${RUNNER_TEMP}/kubeconfig"
-          chmod 0600 "${RUNNER_TEMP}/kubeconfig"
-          export KUBECONFIG="${RUNNER_TEMP}/kubeconfig"
-          "${RUNNER_TEMP}/kubectl" config set-cluster cluster \
-            --server="${KUBERNETES_API}" \
-            --tls-server-name="${KUBERNETES_TLS_SERVER_NAME}"
-
-      - name: Apply and verify
-        run: |
-          export KUBECONFIG="${RUNNER_TEMP}/kubeconfig"
-          "${RUNNER_TEMP}/kubectl" apply --dry-run=server --validate=false \
-            --filename stack.yaml \
-            --filename netflow.yaml \
-            --filename ingress.yaml \
-            --filename traefik-access.yaml
-          "${RUNNER_TEMP}/kubectl" apply --validate=false \
-            --filename stack.yaml \
-            --filename netflow.yaml \
-            --filename ingress.yaml \
-            --filename traefik-access.yaml
-
-          for attempt in {1..240}; do
-            es_health=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get elasticsearch/elasticsearch --output=jsonpath='{.status.health}')
-            es_nodes=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get elasticsearch/elasticsearch --output=jsonpath='{.status.availableNodes}')
-            kb_health=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get kibana/kibana --output=jsonpath='{.status.health}')
-            kb_nodes=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get kibana/kibana --output=jsonpath='{.status.availableNodes}')
-            beat_health=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get beat/netflow --output=jsonpath='{.status.health}')
-            beat_nodes=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get beat/netflow --output=jsonpath='{.status.availableNodes}')
-            agent_health=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get agent/traefik-access --output=jsonpath='{.status.health}')
-            agent_nodes=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get agent/traefik-access --output=jsonpath='{.status.availableNodes}')
-            agent_expected=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get agent/traefik-access --output=jsonpath='{.status.expectedNodes}')
-            load_balancer_ip=$("${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" get service/netflow --output=jsonpath='{.status.loadBalancer.ingress[0].ip}')
-
-            echo "Check ${attempt}/240: Elasticsearch ${es_health:-unknown} (${es_nodes:-0}/3), Kibana ${kb_health:-unknown} (${kb_nodes:-0}/2), Filebeat ${beat_health:-unknown} (${beat_nodes:-0}/1), Agent ${agent_health:-unknown} (${agent_nodes:-0}/${agent_expected:-0}), LB=${load_balancer_ip:-pending}"
-            if [[ "${es_health}" == "green" \
-                  && "${es_nodes}" == "3" \
-                  && "${kb_health}" == "green" \
-                  && "${kb_nodes}" == "2" \
-                  && "${beat_health}" == "green" \
-                  && "${beat_nodes}" == "1" \
-                  && "${agent_health}" == "green" \
-                  && -n "${agent_expected}" \
-                  && "${agent_expected}" != "0" \
-                  && "${agent_nodes}" == "${agent_expected}" \
-                  && "${load_balancer_ip}" == "192.168.0.20" ]]; then
-              echo "Elastic Stack successfully rolled out"
-              exit 0
-            fi
-
-            sleep 5
-          done
-
-          echo "Elastic Stack did not become ready within twenty minutes" >&2
-          exit 1

.gitea/workflows/validate.yml

@@ -0,0 +1,11 @@
+name: Validate Elastic asset tooling
+on: [push, pull_request]
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Validate scripts and Kibana assets
+      run: |
+        bash -n install-traefik-assets.sh check.sh configure-router.sh
+        jq --exit-status . traefik-access-ilm.json traefik-access-pipeline.json traefik-access-component-template.json traefik-traffic-dashboard.json >/dev/null

README.md

@@ -1,5 +1,11 @@
 # Elastic network-observability stack
 
+The live Kubernetes operator and ECK resources are now owned by Argo CD in
+[homelab-gitops](https://code.brunner.ninja/feedc0de/homelab-gitops/src/branch/main/apps/elastic).
+The Kubernetes manifests in this checkout are historical reference only;
+editing or pushing them does not deploy them. This repository remains active
+for the MikroTik router configuration and Kibana pipeline/dashboard assets.
+
 Elasticsearch, Kibana, a dedicated MikroTik IPFIX collector, and a Traefik
 access-log collector managed by Elastic Cloud on Kubernetes (ECK). Everything
 runs in the `elastic` namespace.
@@ -26,16 +32,16 @@ Traefik JSON access logs --> Elastic Agent on each node
                          Elasticsearch
 ```
 
-- ECK operator `3.5.0`, restricted to managing the `elastic` namespace.
-- Elasticsearch `9.5.2`: three nodes, 12 GiB memory each, and three expandable
+- ECK operator, restricted to managing the `elastic` namespace.
+- Elasticsearch: three nodes, 12 GiB memory each, and three expandable
   60 GiB `rook-ceph-block-ec` volumes (180 GiB / about 193 GB total, including
   Elasticsearch's own shard replicas). The RBD image metadata remains on the
   replicated pool, while bulk data uses the 3+2 erasure-coded pool.
-- Kibana `9.5.2`: two 4 GiB replicas at
+- Kibana: two 4 GiB replicas at
   <https://kibana.brunner.ninja>.
-- Filebeat `9.5.2`: one stateful UDP collector. Keeping a single collector
+- Filebeat: one stateful UDP collector. Keeping a single collector
   ensures all IPFIX templates and their flow packets reach the same process.
-- Elastic Agent `9.5.2`: one pod per Traefik node, reading container access
+- Elastic Agent: one pod per Traefik node, reading container access
   logs into the `logs-traefik.access-default` data stream.
 - All Stack components use Elastic's hardened Wolfi images. The standard
   UBI 10 images require the x86-64-v3 CPU baseline, which the Odroid H3 CPUs do
@@ -50,22 +56,21 @@ Traefik JSON access logs --> Elastic Agent on each node
   retain their operator defaults. PVC capacity is a hard 180 GiB aggregate
   ceiling, but Elasticsearch disk-watermark safeguards will intervene earlier.
 
-## Install
+## Router and Kibana assets
 
 From a host with cluster-admin `kubectl` and Helm access:
 
 ```sh
-./install.sh
 ./configure-router.sh
+./install-traefik-assets.sh
 ./check.sh
 ```
 
-From the current remote location, copy the directory to `dahoam`, deploy from
-there, and configure the router through the SSH jump host:
+To run the external asset tooling from `dahoam`, copy the directory there and
+configure the router through the SSH jump host:
 
 ```sh
 rsync -av elastic/ dahoam:elastic/
-ssh dahoam 'cd elastic && ./install.sh'
 JUMP_HOST=dahoam \
 ROUTER_HOST=admin@192.168.0.254 \
 ROUTER_HOST_KEY_ALIAS=newgw \
@@ -119,37 +124,9 @@ not logged because they frequently contain tokens.
 Run `./check.sh` to inspect readiness, recent collector logs, and the number of
 stored `netflow.log` events.
 
-## Gitea Actions CD
+## Gitea Actions
 
-The ECK operator and initial resources must be bootstrapped once with
-`install.sh`. CI subsequently updates only the committed stack resources; it
-cannot read Secrets, modify RBAC, or change the operator.
-
-Create the restricted deployer and generate its kubeconfig:
-
-```sh
-KUBECTL_SSH_HOST=dahoam \
-SSH_CONFIG="$HOME/.ssh/config" \
-./create-ci-kubeconfig.sh
-```
-
-Create one repository Actions secret:
-
-- `KUBE_CONFIG_BASE64`: the single base64 line printed by the command above.
-
-No Elasticsearch, Kibana, router, DNS, or certificate secret belongs in
-Gitea. ECK and cert-manager create and rotate the live credentials inside the
-cluster.
-
-## Removal
-
-This destroys the indices, Kibana saved objects, credentials, and all three
-PVCs:
-
-```sh
-./uninstall.sh --yes
-```
-
-The script removes the Helm release and namespace but intentionally retains
-the cluster-scoped ECK CRDs. Removing CRDs is unsafe if another ECK deployment
-is ever added to the cluster.
+CI validates the retained router and Kibana asset tooling only. It has no
+Kubernetes credentials and cannot deploy the ECK stack. Argo CD owns its
+Kubernetes desired state; do not manually uninstall its Elasticsearch custom
+resource because its claim deletion policy can remove all three data PVCs.

ci-deployer.yaml

@@ -1,108 +0,0 @@
-# Bootstrap once with an administrator identity. CI can update only the
-# existing Elastic resources and cannot read Secrets or change RBAC.
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: elastic-deployer
-  namespace: elastic
-automountServiceAccountToken: false
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: elastic-deployer
-  namespace: elastic
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-    resourceNames:
-      - netflow-ilm-policy
-    verbs:
-      - get
-      - patch
-      - update
-  - apiGroups:
-      - ""
-    resources:
-      - services
-    resourceNames:
-      - netflow
-    verbs:
-      - get
-      - patch
-      - update
-  - apiGroups:
-      - elasticsearch.k8s.elastic.co
-    resources:
-      - elasticsearches
-    resourceNames:
-      - elasticsearch
-    verbs:
-      - get
-      - patch
-      - update
-  - apiGroups:
-      - kibana.k8s.elastic.co
-    resources:
-      - kibanas
-    resourceNames:
-      - kibana
-    verbs:
-      - get
-      - patch
-      - update
-  - apiGroups:
-      - beat.k8s.elastic.co
-    resources:
-      - beats
-    resourceNames:
-      - netflow
-    verbs:
-      - get
-      - patch
-      - update
-  - apiGroups:
-      - agent.k8s.elastic.co
-    resources:
-      - agents
-    resourceNames:
-      - traefik-access
-    verbs:
-      - get
-      - patch
-      - update
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingresses
-    resourceNames:
-      - kibana
-    verbs:
-      - get
-      - patch
-      - update
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: elastic-deployer
-  namespace: elastic
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: elastic-deployer
-subjects:
-  - kind: ServiceAccount
-    name: elastic-deployer
-    namespace: elastic
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: elastic-deployer-token
-  namespace: elastic
-  annotations:
-    kubernetes.io/service-account.name: elastic-deployer
-type: kubernetes.io/service-account-token

create-ci-kubeconfig.sh

@@ -1,66 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
-namespace=${NAMESPACE:-elastic}
-service_account=${SERVICE_ACCOUNT:-elastic-deployer}
-secret=${TOKEN_SECRET:-elastic-deployer-token}
-ssh_host=${KUBECTL_SSH_HOST:-}
-ssh_config=${SSH_CONFIG:-}
-
-cluster_kubectl() {
-  if [[ -n "${ssh_host}" ]]; then
-    if [[ -n "${ssh_config}" ]]; then
-      ssh -F "${ssh_config}" "${ssh_host}" kubectl "$@"
-    else
-      ssh "${ssh_host}" kubectl "$@"
-    fi
-  else
-    kubectl "$@"
-  fi
-}
-
-if [[ -n "${ssh_host}" ]]; then
-  cluster_kubectl apply --filename=- < "${project_dir}/ci-deployer.yaml" >&2
-else
-  cluster_kubectl apply --filename "${project_dir}/ci-deployer.yaml" >&2
-fi
-
-for attempt in {1..30}; do
-  if cluster_kubectl --namespace "${namespace}" get secret "${secret}" \
-      --output=jsonpath='{.data.token}' 2>/dev/null | grep -q .; then
-    break
-  fi
-  if [[ "${attempt}" == 30 ]]; then
-    echo "Timed out waiting for the service-account token" >&2
-    exit 1
-  fi
-  sleep 1
-done
-
-workdir=$(mktemp --directory)
-trap 'rm -rf "${workdir}"' EXIT
-
-server=$(cluster_kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}')
-cluster_kubectl --namespace "${namespace}" get secret "${secret}" \
-  --output=jsonpath='{.data.ca\.crt}' | base64 --decode > "${workdir}/ca.crt"
-token=$(cluster_kubectl --namespace "${namespace}" get secret "${secret}" \
-  --output=jsonpath='{.data.token}' | base64 --decode)
-
-export KUBECONFIG="${workdir}/config"
-kubectl config set-cluster cluster \
-  --server="${server}" \
-  --certificate-authority="${workdir}/ca.crt" \
-  --embed-certs=true >/dev/null
-kubectl config set-credentials "${service_account}" --token="${token}" >/dev/null
-kubectl config set-context elastic \
-  --cluster=cluster \
-  --user="${service_account}" \
-  --namespace="${namespace}" >/dev/null
-kubectl config use-context elastic >/dev/null
-
-echo "Store the following single line as the Gitea Actions secret KUBE_CONFIG_BASE64." >&2
-echo "Treat it as a password; do not commit it." >&2
-base64 --wrap=0 "${KUBECONFIG}"
-printf '\n'

install.sh

@@ -1,121 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
-namespace=elastic
-operator_version=3.5.0
-
-for command in helm kubectl; do
-  if ! command -v "${command}" >/dev/null 2>&1; then
-    echo "Required command not found: ${command}" >&2
-    exit 1
-  fi
-done
-
-for dependency in storageclass/rook-ceph-block-ec; do
-  if ! kubectl get "${dependency}" >/dev/null 2>&1; then
-    echo "Required Kubernetes object not found: ${dependency}" >&2
-    exit 1
-  fi
-done
-
-helm repo add elastic https://helm.elastic.co --force-update
-helm repo update elastic
-
-helm upgrade --install elastic-operator elastic/eck-operator \
-  --namespace "${namespace}" \
-  --create-namespace \
-  --version "${operator_version}" \
-  --values "${project_dir}/operator-values.yaml" \
-  --wait \
-  --timeout 10m
-
-for crd in \
-  elasticsearches.elasticsearch.k8s.elastic.co \
-  kibanas.kibana.k8s.elastic.co \
-  beats.beat.k8s.elastic.co \
-  agents.agent.k8s.elastic.co; do
-  kubectl wait --for=condition=Established "crd/${crd}" --timeout=2m
-done
-
-kubectl apply --dry-run=server --validate=false \
-  --filename "${project_dir}/stack.yaml" \
-  --filename "${project_dir}/netflow.yaml" \
-  --filename "${project_dir}/ingress.yaml"
-kubectl apply --validate=false \
-  --filename "${project_dir}/stack.yaml" \
-  --filename "${project_dir}/netflow.yaml" \
-  --filename "${project_dir}/ingress.yaml"
-
-for attempt in {1..360}; do
-  es_health=$(kubectl --namespace "${namespace}" get elasticsearch/elasticsearch \
-    --output=jsonpath='{.status.health}')
-  es_nodes=$(kubectl --namespace "${namespace}" get elasticsearch/elasticsearch \
-    --output=jsonpath='{.status.availableNodes}')
-  kb_health=$(kubectl --namespace "${namespace}" get kibana/kibana \
-    --output=jsonpath='{.status.health}')
-  kb_nodes=$(kubectl --namespace "${namespace}" get kibana/kibana \
-    --output=jsonpath='{.status.availableNodes}')
-  beat_health=$(kubectl --namespace "${namespace}" get beat/netflow \
-    --output=jsonpath='{.status.health}')
-  beat_nodes=$(kubectl --namespace "${namespace}" get beat/netflow \
-    --output=jsonpath='{.status.availableNodes}')
-  load_balancer_ip=$(kubectl --namespace "${namespace}" get service/netflow \
-    --output=jsonpath='{.status.loadBalancer.ingress[0].ip}')
-
-  echo "Check ${attempt}/360: Elasticsearch ${es_health:-unknown} (${es_nodes:-0}/3), Kibana ${kb_health:-unknown} (${kb_nodes:-0}/2), Filebeat ${beat_health:-unknown} (${beat_nodes:-0}/1), LB=${load_balancer_ip:-pending}"
-  if [[ "${es_health}" == "green" \
-        && "${es_nodes}" == "3" \
-        && "${kb_health}" == "green" \
-        && "${kb_nodes}" == "2" \
-        && "${beat_health}" == "green" \
-        && "${beat_nodes}" == "1" \
-        && "${load_balancer_ip}" == "192.168.0.20" ]]; then
-    break
-  fi
-
-  if [[ "${attempt}" == "360" ]]; then
-    echo "Timed out waiting for the complete Elastic Stack rollout" >&2
-    exit 1
-  fi
-  sleep 5
-done
-
-"${project_dir}/install-traefik-assets.sh"
-
-kubectl apply --dry-run=server --validate=false \
-  --filename "${project_dir}/traefik-access.yaml"
-kubectl apply --validate=false \
-  --filename "${project_dir}/traefik-access.yaml"
-
-for attempt in {1..240}; do
-  agent_health=$(kubectl --namespace "${namespace}" get agent/traefik-access \
-    --output=jsonpath='{.status.health}')
-  agent_nodes=$(kubectl --namespace "${namespace}" get agent/traefik-access \
-    --output=jsonpath='{.status.availableNodes}')
-  agent_expected=$(kubectl --namespace "${namespace}" get agent/traefik-access \
-    --output=jsonpath='{.status.expectedNodes}')
-
-  echo "Agent check ${attempt}/240: ${agent_health:-unknown} (${agent_nodes:-0}/${agent_expected:-0})"
-  if [[ "${agent_health}" == "green" \
-        && -n "${agent_expected}" \
-        && "${agent_expected}" != "0" \
-        && "${agent_nodes}" == "${agent_expected}" ]]; then
-    break
-  fi
-
-  if [[ "${attempt}" == "240" ]]; then
-    echo "Timed out waiting for the Traefik access-log agents" >&2
-    exit 1
-  fi
-  sleep 5
-done
-
-echo
-echo "Elastic Stack is ready in namespace ${namespace}."
-echo "Kibana: https://kibana.brunner.ninja"
-echo "NetFlow collector: ${load_balancer_ip}:2055/UDP"
-echo "Traefik access logs: logs-traefik.access-default"
-echo "Retrieve the Kibana 'elastic' password with:"
-echo "kubectl get secret elasticsearch-es-elastic-user -n ${namespace} -o go-template='{{.data.elastic | base64decode}}{{\"\\n\"}}'"

uninstall.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-namespace=elastic
-
-if [[ "${1:-}" != "--yes" ]]; then
-  echo "This permanently deletes namespace ${namespace}, all Elasticsearch data, PVCs, credentials, and Kibana objects." >&2
-  echo "Run $0 --yes to continue." >&2
-  exit 1
-fi
-
-kubectl delete --ignore-not-found \
-  --filename "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/ingress.yaml" \
-  --filename "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/traefik-access.yaml" \
-  --filename "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/netflow.yaml" \
-  --filename "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/stack.yaml"
-
-helm uninstall elastic-operator --namespace "${namespace}" --wait || true
-kubectl delete namespace "${namespace}" --ignore-not-found --wait=true
-
-echo "Deleted the Elastic Stack and namespace ${namespace}."
-echo "ECK CRDs are intentionally retained because they are cluster-scoped."