From ac8d83a0348404ba2f2ae77502778c15470ce370 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Mon, 26 Oct 2026 07:24:04 +0100 Subject: [PATCH] Set Elastic log retention to 14 days --- README.md | 11 +++++++---- install-traefik-assets.sh | 4 ++-- traefik-access-component-template.json | 2 +- traefik-access-ilm.json | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 345a91e..324afb4 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ MikroTik newgw -- IPFIX/UDP 2055 --> MetalLB 192.168.0.20 v Filebeat NetFlow module | - GeoIP, ASN, ECS and 30-day ILM + GeoIP, ASN, ECS and 14-day ILM | v Elasticsearch <--> Kibana Maps @@ -42,15 +42,18 @@ Traefik JSON access logs --> Elastic Agent on each node - Filebeat: one stateful UDP collector. Keeping a single collector ensures all IPFIX templates and their flow packets reach the same process. - Elastic Agent: one pod per Traefik node, reading container access - logs into the `logs-traefik.access-default` data stream. + logs into the `logs-traefik.access-homelab` data stream. The old + `logs-traefik.access-default` index is retained only while its remaining + documents age out under the 14-day cleanup job. - All Stack components use Elastic's hardened Wolfi images. The standard UBI 10 images require the x86-64-v3 CPU baseline, which the Odroid H3 CPUs do not provide; the Wolfi images were verified on an Odroid before deployment. - Traefik serves Kibana with the shared `*.brunner.ninja` certificate from its default TLSStore. Elasticsearch transport and HTTP certificates remain ECK-managed and private. -- Flow indices are deleted after 30 days. Traefik access indices roll over - daily or at 30 GiB per primary shard and are deleted after 90 days. +- Flow and Traefik access indices roll over daily or at 30 GiB per primary + shard and are deleted after 14 days. The legacy Traefik index is pruned by + timestamp until its remaining documents age out. - NetFlow and Traefik access-log indices have zero Elasticsearch shard replicas: these are rebuildable telemetry on a Ceph 3+2 EC pool. Other system indices retain their operator defaults. PVC capacity is a hard 180 GiB aggregate diff --git a/install-traefik-assets.sh b/install-traefik-assets.sh index e47943c..bce28c1 100755 --- a/install-traefik-assets.sh +++ b/install-traefik-assets.sh @@ -63,7 +63,7 @@ kubectl exec --namespace "${namespace}" "${pod}" --container elasticsearch -- \ "http://kibana-kb-http.elastic.svc:5601/api/fleet/epm/packages/traefik/${package_version}" echo -elastic_api PUT '_ilm/policy/traefik-access-90d' \ +elastic_api PUT '_ilm/policy/traefik-access-14d' \ "${project_dir}/traefik-access-ilm.json" elastic_api PUT '_ingest/pipeline/logs-traefik.access@custom' \ "${project_dir}/traefik-access-pipeline.json" @@ -74,4 +74,4 @@ elastic_api PUT 'logs-traefik.access-*/_settings?allow_no_indices=true' \ kibana_import "${project_dir}/traefik-traffic-dashboard.json" unset password -echo "Installed Traefik ingest pipelines, dashboards, subnet mappings, and 90-day retention." +echo "Installed Traefik ingest pipelines, dashboards, subnet mappings, and 14-day retention." diff --git a/traefik-access-component-template.json b/traefik-access-component-template.json index aadd4b2..5f35990 100644 --- a/traefik-access-component-template.json +++ b/traefik-access-component-template.json @@ -1,7 +1,7 @@ { "template": { "settings": { - "index.lifecycle.name": "traefik-access-90d", + "index.lifecycle.name": "traefik-access-14d", "index.number_of_replicas": 0, "index.final_pipeline": "logs-traefik.access@custom" }, diff --git a/traefik-access-ilm.json b/traefik-access-ilm.json index fe2bbf6..599dd3a 100644 --- a/traefik-access-ilm.json +++ b/traefik-access-ilm.json @@ -10,7 +10,7 @@ } }, "delete": { - "min_age": "90d", + "min_age": "14d", "actions": { "delete": {} }