Set Elastic log retention to 14 days

apps/elastic/README.md

@@ -16,3 +16,9 @@ The MikroTik IPFIX export and the Kibana dashboard/pipeline/ILM API assets are
 outside Kubernetes and remain managed by the historical `elastic` repository's
 router and asset scripts. The Kubernetes resource source of truth is here; the
 old repository's direct Kubernetes deploy workflow must not run after cutover.
+
+Homelab NetFlow and Traefik access logs retain 14 days. NetFlow uses its ILM
+policy. The `elasticsearch-log-retention` CronJob prunes expired timestamps
+from the historical Traefik index and Elastic Agent log index. New Traefik
+events use a separate data stream with the 14-day ILM policy, avoiding the
+unmanaged legacy index.

apps/elastic/resources/kustomization.yaml

@@ -4,4 +4,5 @@ resources:
 - stack.yaml
 - netflow.yaml
 - traefik-access.yaml
+- retention.yaml
 - ingress.yaml

apps/elastic/resources/netflow.yaml

@@ -20,7 +20,7 @@ data:
             }
           },
           "delete": {
-            "min_age": "30d",
+            "min_age": "14d",
             "actions": {
               "delete": {}
             }

apps/elastic/resources/retention.yaml

@@ -0,0 +1,83 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: elasticsearch-log-retention
+  namespace: elastic
+  labels:
+    app.kubernetes.io/name: elasticsearch-log-retention
+    app.kubernetes.io/part-of: elastic-stack
+spec:
+  schedule: "0 13 * * *"
+  timeZone: Europe/Vienna
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 3600
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 14400
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: elasticsearch-log-retention
+            app.kubernetes.io/part-of: elastic-stack
+        spec:
+          automountServiceAccountToken: false
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1000
+            runAsGroup: 1000
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: prune-expired-logs
+              image: docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.5.5
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - -ec
+              args:
+                - |
+                  response=$(curl --silent --show-error --fail --max-time 14400 \
+                    --cacert /tls/ca.crt \
+                    --user "elastic:${ELASTIC_PASSWORD}" \
+                    --header 'Content-Type: application/json' \
+                    --data-binary '{"query":{"range":{"@timestamp":{"lt":"now-14d"}}}}' \
+                    'https://elasticsearch-es-http.elastic.svc:9200/logs-traefik.access-default,logs-elastic_agent-default/_delete_by_query?conflicts=proceed&refresh=true&requests_per_second=5000')
+                  printf '%s\n' "$response"
+                  printf '%s' "$response" | grep -Eq '"failures"[[:space:]]*:[[:space:]]*\[\]' || {
+                    echo "Elasticsearch reported delete-by-query failures" >&2
+                    exit 1
+                  }
+              env:
+                - name: ELASTIC_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: elasticsearch-es-elastic-user
+                      key: elastic
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 128Mi
+                limits:
+                  cpu: 500m
+                  memory: 512Mi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop:
+                    - ALL
+              volumeMounts:
+                - name: elasticsearch-http-ca
+                  mountPath: /tls
+                  readOnly: true
+          volumes:
+            - name: elasticsearch-http-ca
+              secret:
+                secretName: elasticsearch-es-http-certs-public
+                items:
+                  - key: ca.crt
+                    path: ca.crt

apps/elastic/resources/traefik-access.yaml

@@ -25,7 +25,7 @@ spec:
         type: filestream
         use_output: default
         data_stream:
-          namespace: default
+          namespace: homelab
         streams:
           - id: traefik-access-json
             data_stream: