diff --git a/apps/elastic/README.md b/apps/elastic/README.md new file mode 100644 index 0000000..3448f15 --- /dev/null +++ b/apps/elastic/README.md @@ -0,0 +1,18 @@ +# Elastic observability + +`elastic-operator` adopts the existing upstream ECK 3.5.0 Helm release in +`elastic`. The separate `elastic` Application owns the existing Elasticsearch, +Kibana, Beat, and Agent custom resources plus their cluster-specific ConfigMap, +Service, and Ingress. ECK alone owns the generated StatefulSet, Deployments, +DaemonSet, Services, and credentials. + +The three Elasticsearch data PVCs predate Argo and must retain their UIDs and +bound volumes. Both Applications deliberately disable pruning. In particular, +the existing Elasticsearch custom resource specifies +`DeleteOnScaledownAndClusterDeletion`; deleting it can delete its data claims. +Do not delete or rename it as part of a GitOps cleanup. + +The MikroTik IPFIX export and the Kibana dashboard/pipeline/ILM API assets are +outside Kubernetes and remain managed by the historical `elastic` repository's +router and asset scripts. The Kubernetes resource source of truth is here; the +old repository's direct Kubernetes deploy workflow must not run after cutover. diff --git a/apps/elastic/operator-values.yaml b/apps/elastic/operator-values.yaml new file mode 100644 index 0000000..3159695 --- /dev/null +++ b/apps/elastic/operator-values.yaml @@ -0,0 +1,20 @@ +managedNamespaces: + - elastic + +installCRDs: true +createClusterScopedResources: true + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1 + memory: 1Gi + +config: + metrics: + port: "0" + +telemetry: + disabled: true diff --git a/apps/elastic/resources/ingress.yaml b/apps/elastic/resources/ingress.yaml new file mode 100644 index 0000000..17e173d --- /dev/null +++ b/apps/elastic/resources/ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: kibana + namespace: elastic + labels: + app.kubernetes.io/name: kibana + app.kubernetes.io/part-of: elastic-stack + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + rules: + - host: kibana.brunner.ninja + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kibana-kb-http + port: + number: 5601 diff --git a/apps/elastic/resources/kustomization.yaml b/apps/elastic/resources/kustomization.yaml new file mode 100644 index 0000000..e4f7a33 --- /dev/null +++ b/apps/elastic/resources/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- stack.yaml +- netflow.yaml +- traefik-access.yaml +- ingress.yaml diff --git a/apps/elastic/resources/netflow.yaml b/apps/elastic/resources/netflow.yaml new file mode 100644 index 0000000..b9b7433 --- /dev/null +++ b/apps/elastic/resources/netflow.yaml @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: netflow-ilm-policy + namespace: elastic + labels: + app.kubernetes.io/name: netflow + app.kubernetes.io/part-of: elastic-stack +data: + policy.json: | + { + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "1d", + "max_primary_shard_size": "30gb" + } + } + }, + "delete": { + "min_age": "30d", + "actions": { + "delete": {} + } + } + } + } + } +--- +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: netflow + namespace: elastic + labels: + app.kubernetes.io/name: netflow + app.kubernetes.io/part-of: elastic-stack +spec: + type: filebeat + version: 9.5.5 + image: docker.elastic.co/beats/filebeat-wolfi:9.5.5 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + filebeat.modules: + - module: netflow + log: + enabled: true + var: + netflow_host: 0.0.0.0 + netflow_port: 2055 + expiration_timeout: 30m + queue_size: 32768 + internal_networks: + - private + processors: + - add_fields: + target: observer + fields: + name: newgw + vendor: MikroTik + product: RouterOS + type: firewall + setup.dashboards.enabled: true + setup.dashboards.retry.enabled: true + setup.dashboards.retry.interval: 10s + setup.dashboards.retry.maximum: 0 + setup.ilm.enabled: true + setup.ilm.policy_name: netflow-30d + setup.ilm.policy_file: /usr/share/filebeat/ilm/policy.json + setup.ilm.overwrite: true + setup.template.settings: + index.number_of_replicas: 0 + queue.mem.events: 16384 + queue.mem.flush.min_events: 1024 + queue.mem.flush.timeout: 1s + max_procs: 2 + logging.level: info + deployment: + podTemplate: + metadata: + labels: + app.kubernetes.io/name: netflow + app.kubernetes.io/part-of: elastic-stack + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: filebeat + securityContext: + runAsUser: 0 + ports: + - name: netflow + containerPort: 2055 + protocol: UDP + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + volumeMounts: + - name: ilm-policy + mountPath: /usr/share/filebeat/ilm + readOnly: true + volumes: + - name: ilm-policy + configMap: + name: netflow-ilm-policy +--- +apiVersion: v1 +kind: Service +metadata: + name: netflow + namespace: elastic + labels: + app.kubernetes.io/name: netflow + app.kubernetes.io/part-of: elastic-stack + annotations: + metallb.io/loadBalancerIPs: "192.168.0.20" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + beat.k8s.elastic.co/name: netflow + ports: + - name: netflow + port: 2055 + targetPort: netflow + protocol: UDP diff --git a/apps/elastic/resources/stack.yaml b/apps/elastic/resources/stack.yaml new file mode 100644 index 0000000..e6c188f --- /dev/null +++ b/apps/elastic/resources/stack.yaml @@ -0,0 +1,126 @@ +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch + namespace: elastic + labels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/part-of: elastic-stack +spec: + version: 9.5.5 + image: docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.5.5 + volumeClaimDeletePolicy: DeleteOnScaledownAndClusterDeletion + nodeSets: + - name: data + count: 3 + config: + node.store.allow_mmap: false + podTemplate: + metadata: + labels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/part-of: elastic-stack + spec: + terminationGracePeriodSeconds: 180 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - odroid-h3-00 + - odroid-h3-01 + - odroid-h3-02 + - odroid-h3-03 + - odroid-h3-04 + - odroid-h3-05 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + elasticsearch.k8s.elastic.co/cluster-name: elasticsearch + containers: + - name: elasticsearch + resources: + requests: + cpu: "2" + memory: 12Gi + limits: + cpu: "4" + memory: 12Gi + volumeClaimTemplates: + - metadata: + name: elasticsearch-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: rook-ceph-block-ec + resources: + requests: + storage: 60Gi + http: + service: + spec: + type: ClusterIP +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana + namespace: elastic + labels: + app.kubernetes.io/name: kibana + app.kubernetes.io/part-of: elastic-stack +spec: + version: 9.5.5 + image: docker.elastic.co/kibana/kibana-wolfi:9.5.5 + count: 2 + elasticsearchRef: + name: elasticsearch + config: + server.publicBaseUrl: https://kibana.brunner.ninja + xpack.security.secureCookies: true + http: + tls: + selfSignedCertificate: + disabled: true + podTemplate: + metadata: + labels: + app.kubernetes.io/name: kibana + app.kubernetes.io/part-of: elastic-stack + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - odroid-h3-00 + - odroid-h3-01 + - odroid-h3-02 + - odroid-h3-03 + - odroid-h3-04 + - odroid-h3-05 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + kibana.k8s.elastic.co/name: kibana + containers: + - name: kibana + resources: + requests: + cpu: 500m + memory: 4Gi + limits: + cpu: "2" + memory: 4Gi diff --git a/apps/elastic/resources/traefik-access.yaml b/apps/elastic/resources/traefik-access.yaml new file mode 100644 index 0000000..4c437d6 --- /dev/null +++ b/apps/elastic/resources/traefik-access.yaml @@ -0,0 +1,83 @@ +apiVersion: agent.k8s.elastic.co/v1alpha1 +kind: Agent +metadata: + name: traefik-access + namespace: elastic + labels: + app.kubernetes.io/name: traefik-access + app.kubernetes.io/part-of: elastic-stack +spec: + version: 9.5.5 + image: docker.elastic.co/elastic-agent/elastic-agent-wolfi:9.5.5 + mode: standalone + elasticsearchRefs: + - name: elasticsearch + config: + agent: + monitoring: + enabled: true + use_output: default + logs: true + metrics: true + inputs: + - id: traefik-access + name: Traefik access logs + type: filestream + use_output: default + data_stream: + namespace: default + streams: + - id: traefik-access-json + data_stream: + dataset: traefik.access + type: logs + paths: + - /var/log/containers/traefik-*_default_traefik-*.log + parsers: + - container: + stream: stdout + format: auto + prospector.scanner.symlinks: true + tags: + - forwarded + processors: + - drop_event: + when: + not: + regexp: + message: '^\{' + daemonSet: + podTemplate: + metadata: + labels: + app.kubernetes.io/name: traefik-access + app.kubernetes.io/part-of: elastic-stack + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsUser: 0 + containers: + - name: agent + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: container-logs + mountPath: /var/log/containers + readOnly: true + - name: pod-logs + mountPath: /var/log/pods + readOnly: true + volumes: + - name: container-logs + hostPath: + path: /var/log/containers + type: Directory + - name: pod-logs + hostPath: + path: /var/log/pods + type: Directory diff --git a/bootstrap/elastic-operator.yaml b/bootstrap/elastic-operator.yaml new file mode 100644 index 0000000..725cb41 --- /dev/null +++ b/bootstrap/elastic-operator.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: elastic-operator + namespace: argocd +spec: + project: elastic + sources: + - repoURL: https://helm.elastic.co + chart: eck-operator + targetRevision: 3.5.0 + helm: + releaseName: elastic-operator + valueFiles: + - $values/apps/elastic/operator-values.yaml + - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git + targetRevision: main + ref: values + destination: + server: https://kubernetes.default.svc + namespace: elastic + syncPolicy: + automated: + enabled: true + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=false diff --git a/bootstrap/elastic-project.yaml b/bootstrap/elastic-project.yaml new file mode 100644 index 0000000..ec24064 --- /dev/null +++ b/bootstrap/elastic-project.yaml @@ -0,0 +1,51 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: elastic + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "-1" +spec: + description: ECK operator and this cluster's Elastic observability resources + sourceRepos: + - https://helm.elastic.co + - https://code.brunner.ninja/feedc0de/homelab-gitops.git + destinations: + - namespace: elastic + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + namespaceResourceWhitelist: + - group: "" + kind: ConfigMap + - group: "" + kind: Secret + - group: "" + kind: Service + - group: "" + kind: ServiceAccount + - group: apps + kind: StatefulSet + - group: networking.k8s.io + kind: Ingress + - group: rbac.authorization.k8s.io + kind: Role + - group: rbac.authorization.k8s.io + kind: RoleBinding + - group: elasticsearch.k8s.elastic.co + kind: Elasticsearch + - group: kibana.k8s.elastic.co + kind: Kibana + - group: beat.k8s.elastic.co + kind: Beat + - group: agent.k8s.elastic.co + kind: Agent diff --git a/bootstrap/elastic.yaml b/bootstrap/elastic.yaml new file mode 100644 index 0000000..a5ec4a8 --- /dev/null +++ b/bootstrap/elastic.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: elastic + namespace: argocd +spec: + project: elastic + source: + repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git + targetRevision: main + path: apps/elastic/resources + destination: + server: https://kubernetes.default.svc + namespace: elastic + syncPolicy: + automated: + enabled: true + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=false diff --git a/bootstrap/kustomization.yaml b/bootstrap/kustomization.yaml index 98560cd..f85ea6e 100644 --- a/bootstrap/kustomization.yaml +++ b/bootstrap/kustomization.yaml @@ -76,3 +76,6 @@ resources: - nextcloud-talk-hpb.yaml - nextcloud-clamav.yaml - nextcloud-notify-push.yaml +- elastic-project.yaml +- elastic-operator.yaml +- elastic.yaml