diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 4dbe27c..0d5149b 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -24,19 +24,19 @@ jobs: bash -n install.sh install-traefik-assets.sh check.sh configure-router.sh uninstall.sh create-ci-kubeconfig.sh jq --exit-status . traefik-access-ilm.json traefik-access-pipeline.json traefik-access-component-template.json traefik-traffic-dashboard.json >/dev/null docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c" \ + "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ -strict -summary -ignore-missing-schemas < stack.yaml docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c" \ + "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ -strict -summary -ignore-missing-schemas < netflow.yaml docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c" \ + "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ -strict -summary -ignore-missing-schemas < ingress.yaml docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c" \ + "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ -strict -summary -ignore-missing-schemas < traefik-access.yaml docker run --rm --interactive \ - "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c" \ + "ghcr.io/yannh/kubeconform:${KUBECONFORM_VERSION}" \ -strict -summary < ci-deployer.yaml deploy: @@ -53,10 +53,6 @@ jobs: curl --fail --silent --show-error --location \ --output "${RUNNER_TEMP}/kubectl" \ "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - curl --fail --silent --show-error --location \ - --output "${RUNNER_TEMP}/kubectl.sha256" \ - "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" - printf '%s %s\n' "$(cat "${RUNNER_TEMP}/kubectl.sha256")" "${RUNNER_TEMP}/kubectl" | sha256sum --check chmod 0700 "${RUNNER_TEMP}/kubectl" - name: Configure Kubernetes access diff --git a/README.md b/README.md index 032f0d6..19b66e8 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,9 @@ Traefik JSON access logs --> Elastic Agent on each node - ECK operator `3.5.0`, restricted to managing the `elastic` namespace. - Elasticsearch `9.5.2`: three nodes, 12 GiB memory each, and three expandable - 200 GiB `rook-ceph-block` volumes. + 60 GiB `rook-ceph-block-ec` volumes (180 GiB / about 193 GB total, including + Elasticsearch's own shard replicas). The RBD image metadata remains on the + replicated pool, while bulk data uses the 3+2 erasure-coded pool. - Kibana `9.5.2`: two 4 GiB replicas at . - Filebeat `9.5.2`: one stateful UDP collector. Keeping a single collector @@ -43,6 +45,10 @@ Traefik JSON access logs --> Elastic Agent on each node ECK-managed and private. - Flow indices are deleted after 30 days. Traefik access indices roll over daily or at 30 GiB per primary shard and are deleted after 90 days. +- NetFlow and Traefik access-log indices have zero Elasticsearch shard replicas: + these are rebuildable telemetry on a Ceph 3+2 EC pool. Other system indices + retain their operator defaults. PVC capacity is a hard 180 GiB aggregate + ceiling, but Elasticsearch disk-watermark safeguards will intervene earlier. ## Install diff --git a/install-traefik-assets.sh b/install-traefik-assets.sh index aa518c9..e47943c 100755 --- a/install-traefik-assets.sh +++ b/install-traefik-assets.sh @@ -17,16 +17,23 @@ elastic_api() { local method=$1 local path=$2 local input=$3 + local input_data - kubectl exec --stdin --namespace "${namespace}" "${pod}" --container elasticsearch -- \ + if [[ -f "${input}" ]]; then + input_data=$(<"${input}") + else + input_data=${input} + fi + + printf '%s' "${input_data}" | kubectl exec --stdin --namespace "${namespace}" "${pod}" --container elasticsearch -- \ curl --silent --show-error --fail \ + --retry 8 --retry-delay 5 --retry-max-time 600 --max-time 120 \ --request "${method}" \ --cacert /usr/share/elasticsearch/config/http-certs/ca.crt \ --user "elastic:${password}" \ --header 'Content-Type: application/json' \ --data-binary @- \ - "https://elasticsearch-es-http.elastic.svc:9200/${path}" \ - < "${input}" + "https://elasticsearch-es-http.elastic.svc:9200/${path}" echo } @@ -47,6 +54,7 @@ kibana_import() { echo "Installing the official Elastic Traefik integration ${package_version}..." kubectl exec --namespace "${namespace}" "${pod}" --container elasticsearch -- \ curl --silent --show-error --fail \ + --retry 8 --retry-delay 5 --retry-max-time 600 --max-time 120 \ --request POST \ --user "elastic:${password}" \ --header 'kbn-xsrf: true' \ @@ -61,6 +69,8 @@ elastic_api PUT '_ingest/pipeline/logs-traefik.access@custom' \ "${project_dir}/traefik-access-pipeline.json" elastic_api PUT '_component_template/logs-traefik.access@custom' \ "${project_dir}/traefik-access-component-template.json" +elastic_api PUT 'logs-traefik.access-*/_settings?allow_no_indices=true' \ + '{"index":{"number_of_replicas":0}}' kibana_import "${project_dir}/traefik-traffic-dashboard.json" unset password diff --git a/install.sh b/install.sh index 22aec13..a16d705 100755 --- a/install.sh +++ b/install.sh @@ -13,7 +13,7 @@ for command in helm kubectl; do fi done -for dependency in storageclass/rook-ceph-block; do +for dependency in storageclass/rook-ceph-block-ec; do if ! kubectl get "${dependency}" >/dev/null 2>&1; then echo "Required Kubernetes object not found: ${dependency}" >&2 exit 1 diff --git a/netflow.yaml b/netflow.yaml index 062c65e..4ca4b8d 100644 --- a/netflow.yaml +++ b/netflow.yaml @@ -73,6 +73,8 @@ spec: setup.ilm.policy_name: netflow-30d setup.ilm.policy_file: /usr/share/filebeat/ilm/policy.json setup.ilm.overwrite: true + setup.template.settings: + index.number_of_replicas: 0 queue.mem.events: 16384 queue.mem.flush.min_events: 1024 queue.mem.flush.timeout: 1s diff --git a/stack.yaml b/stack.yaml index 50d1405..5be1815 100644 --- a/stack.yaml +++ b/stack.yaml @@ -58,10 +58,10 @@ spec: spec: accessModes: - ReadWriteOnce - storageClassName: rook-ceph-block + storageClassName: rook-ceph-block-ec resources: requests: - storage: 200Gi + storage: 60Gi http: service: spec: diff --git a/traefik-access-component-template.json b/traefik-access-component-template.json index 2f1e57a..aadd4b2 100644 --- a/traefik-access-component-template.json +++ b/traefik-access-component-template.json @@ -2,6 +2,7 @@ "template": { "settings": { "index.lifecycle.name": "traefik-access-90d", + "index.number_of_replicas": 0, "index.final_pipeline": "logs-traefik.access@custom" }, "mappings": {