diff --git a/apps/elastic/README.md b/apps/elastic/README.md index 3448f15..d6aaf82 100644 --- a/apps/elastic/README.md +++ b/apps/elastic/README.md @@ -16,3 +16,9 @@ The MikroTik IPFIX export and the Kibana dashboard/pipeline/ILM API assets are outside Kubernetes and remain managed by the historical `elastic` repository's router and asset scripts. The Kubernetes resource source of truth is here; the old repository's direct Kubernetes deploy workflow must not run after cutover. + +Homelab NetFlow and Traefik access logs retain 14 days. NetFlow uses its ILM +policy. The `elasticsearch-log-retention` CronJob prunes expired timestamps +from the historical Traefik index and Elastic Agent log index. New Traefik +events use a separate data stream with the 14-day ILM policy, avoiding the +unmanaged legacy index. diff --git a/apps/elastic/resources/kustomization.yaml b/apps/elastic/resources/kustomization.yaml index e4f7a33..55d0b30 100644 --- a/apps/elastic/resources/kustomization.yaml +++ b/apps/elastic/resources/kustomization.yaml @@ -4,4 +4,5 @@ resources: - stack.yaml - netflow.yaml - traefik-access.yaml +- retention.yaml - ingress.yaml diff --git a/apps/elastic/resources/netflow.yaml b/apps/elastic/resources/netflow.yaml index b9b7433..c0289a4 100644 --- a/apps/elastic/resources/netflow.yaml +++ b/apps/elastic/resources/netflow.yaml @@ -20,7 +20,7 @@ data: } }, "delete": { - "min_age": "30d", + "min_age": "14d", "actions": { "delete": {} } diff --git a/apps/elastic/resources/retention.yaml b/apps/elastic/resources/retention.yaml new file mode 100644 index 0000000..59bbeb5 --- /dev/null +++ b/apps/elastic/resources/retention.yaml @@ -0,0 +1,83 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: elasticsearch-log-retention + namespace: elastic + labels: + app.kubernetes.io/name: elasticsearch-log-retention + app.kubernetes.io/part-of: elastic-stack +spec: + schedule: "0 13 * * *" + timeZone: Europe/Vienna + concurrencyPolicy: Forbid + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 14400 + template: + metadata: + labels: + app.kubernetes.io/name: elasticsearch-log-retention + app.kubernetes.io/part-of: elastic-stack + spec: + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: prune-expired-logs + image: docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.5.5 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -ec + args: + - | + response=$(curl --silent --show-error --fail --max-time 14400 \ + --cacert /tls/ca.crt \ + --user "elastic:${ELASTIC_PASSWORD}" \ + --header 'Content-Type: application/json' \ + --data-binary '{"query":{"range":{"@timestamp":{"lt":"now-14d"}}}}' \ + 'https://elasticsearch-es-http.elastic.svc:9200/logs-traefik.access-default,logs-elastic_agent-default/_delete_by_query?conflicts=proceed&refresh=true&requests_per_second=5000') + printf '%s\n' "$response" + printf '%s' "$response" | grep -Eq '"failures"[[:space:]]*:[[:space:]]*\[\]' || { + echo "Elasticsearch reported delete-by-query failures" >&2 + exit 1 + } + env: + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch-es-elastic-user + key: elastic + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: elasticsearch-http-ca + mountPath: /tls + readOnly: true + volumes: + - name: elasticsearch-http-ca + secret: + secretName: elasticsearch-es-http-certs-public + items: + - key: ca.crt + path: ca.crt diff --git a/apps/elastic/resources/traefik-access.yaml b/apps/elastic/resources/traefik-access.yaml index 4c437d6..a2caa53 100644 --- a/apps/elastic/resources/traefik-access.yaml +++ b/apps/elastic/resources/traefik-access.yaml @@ -25,7 +25,7 @@ spec: type: filestream use_output: default data_stream: - namespace: default + namespace: homelab streams: - id: traefik-access-json data_stream: