Adopt Prometheus and track kubeadm metrics config

2026-10-21 18:43:06 +02:00
parent 74c4c6bc4e
commit 4fb8b63986
12 changed files with 456 additions and 0 deletions
@@ -0,0 +1,19 @@
+# Prometheus
+
+Argo CD installs the upstream `kube-prometheus-stack` Helm chart at the
+version selected in `bootstrap/prometheus.yaml`. `values.yaml` contains only
+this cluster's intentional settings. The chart owns Prometheus, Alertmanager,
+their supporting workloads, and chart-provided monitoring resources.
+
+The `resources/` directory contains cluster-specific scrape endpoints and
+alert rules that are not chart defaults. It does not contain application
+workload templates.
+
+Prometheus TSDB uses the existing 50Gi `rook-ceph-block` claim. During adoption,
+preserve claim
+`prometheus-prometheus-kube-prometheus-prometheus-db-prometheus-prometheus-kube-prometheus-prometheus-0`
+and PV `pvc-dfeec228-10e1-42f2-af69-ebcac00f5ddc`; do not delete or recreate it.
+
+`kubeadm` control-plane metrics listener and host firewall settings are tracked
+separately under `cluster-config/kubeadm-monitoring/`. Argo CD does not manage
+those host files or static-pod manifests.
@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: external-immich-8082
+  namespace: prometheus
+  labels:
+    app: external-immich-service
+    port: "8082"
+spec:
+  ports:
+    - name: metrics
+      port: 8082
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: external-immich-8082
+  namespace: prometheus
+subsets:
+  - addresses:
+      - ip: 192.168.0.2
+    ports:
+      - name: metrics
+        port: 8082
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: external-immich-8083
+  namespace: prometheus
+  labels:
+    app: external-immich-service
+    port: "8083"
+spec:
+  ports:
+    - name: metrics
+      port: 8083
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: external-immich-8083
+  namespace: prometheus
+subsets:
+  - addresses:
+      - ip: 192.168.0.2
+    ports:
+      - name: metrics
+        port: 8083
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: external-immich-service
+  namespace: prometheus
+spec:
+  selector:
+    matchLabels:
+      app: external-immich-service
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
@@ -0,0 +1,29 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: mikrotik-storage
+  namespace: prometheus
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: mikrotik-storage
+      rules:
+        - alert: MikroTikStorageLow
+          expr: |-
+            mktxp_system_free_hdd_space < 131072
+              and mktxp_system_free_hdd_space >= 65536
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "MikroTik flash is running low on {{ $labels.routerboard_name }}"
+            description: "{{ $labels.routerboard_name }} ({{ $labels.board_name }}) has only {{ $value | humanize1024 }}B of writable flash free. Configuration changes and upgrades may soon fail."
+        - alert: MikroTikStorageCritical
+          expr: mktxp_system_free_hdd_space < 65536
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "MikroTik flash is critically low on {{ $labels.routerboard_name }}"
+            description: "{{ $labels.routerboard_name }} ({{ $labels.board_name }}) has only {{ $value | humanize1024 }}B of writable flash free. Configuration changes and upgrades are at immediate risk."
@@ -0,0 +1,41 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ceph-rbd-volume-capacity
+  namespace: prometheus
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: ceph-rbd-volume-capacity
+      rules:
+        - alert: CephRBDPersistentVolumeFillingUp
+          expr: |-
+            (
+              1 - kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"}
+                / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics"}
+            )
+            * on(namespace,persistentvolumeclaim) group_left(storageclass,volumename)
+              kube_persistentvolumeclaim_info{storageclass=~"rook-ceph-block.*"}
+            > 0.80
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Ceph RBD PVC is more than 80% full"
+            description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is {{ $value | humanizePercentage }} used."
+        - alert: CephRBDPersistentVolumeFillingUp
+          expr: |-
+            (
+              1 - kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"}
+                / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics"}
+            )
+            * on(namespace,persistentvolumeclaim) group_left(storageclass,volumename)
+              kube_persistentvolumeclaim_info{storageclass=~"rook-ceph-block.*"}
+            > 0.90
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Ceph RBD PVC is more than 90% full"
+            description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is {{ $value | humanizePercentage }} used."
@@ -0,0 +1,92 @@
+grafana:
+  enabled: false
+
+defaultRules:
+  rules:
+    kubeProxy: false
+
+alertmanager:
+  ingress:
+    enabled: true
+    ingressClassName: traefik
+    annotations:
+      traefik.ingress.kubernetes.io/router.entrypoints: websecure
+      traefik.ingress.kubernetes.io/router.middlewares: default-authentik@kubernetescrd
+    hosts:
+      - alertmanager.brunner.ninja
+    paths:
+      - /
+    pathType: ImplementationSpecific
+
+  alertmanagerSpec:
+    externalUrl: https://alertmanager.brunner.ninja
+    routePrefix: /
+
+  config:
+    global:
+      resolve_timeout: 5m
+      smtp_smarthost: smtp-relay.gmail.com:587
+      smtp_from: Cluster Alerts <alerts@brunner.ninja>
+      smtp_hello: brunner.ninja
+      smtp_require_tls: true
+    route:
+      group_by:
+        - namespace
+        - alertname
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 6h
+      receiver: cluster-email
+      routes:
+        - receiver: "null"
+          matchers:
+            - alertname =~ "Watchdog|InfoInhibitor"
+        - receiver: "null"
+          matchers:
+            - severity = "info"
+        - receiver: cluster-email
+          matchers:
+            - severity = "critical"
+          repeat_interval: 2h
+          continue: true
+        - receiver: cluster-email
+          matchers:
+            - severity != "critical"
+          continue: true
+        - receiver: cluster-matrix
+    receivers:
+      - name: "null"
+      - name: cluster-email
+        email_configs:
+          - to: Daniel Brunner <daniel@brunner.ninja>
+            send_resolved: true
+            headers:
+              Subject: >-
+                [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}{{ if .CommonLabels.namespace }} in {{ .CommonLabels.namespace }}{{ end }}
+      - name: cluster-matrix
+        webhook_configs:
+          - url: http://matrix-alertmanager-receiver.prometheus.svc.cluster.local:12345/alerts/cluster
+            send_resolved: true
+
+prometheus:
+  ingress:
+    enabled: true
+    hosts:
+      - prometheus.brunner.ninja
+    annotations:
+      kubernetes.io/ingress.class: "traefik"
+      traefik.ingress.kubernetes.io/router.entrypoints: websecure
+  prometheusSpec:
+    retention: "15d"
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: rook-ceph-block
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 50Gi
+    serviceMonitorSelectorNilUsesHelmValues: false
+    serviceMonitorSelector: {}
+    ruleSelectorNilUsesHelmValues: false
+    ruleSelector: {}
@@ -60,6 +60,8 @@ resources:
 - tautulli.yaml
 - uptime-kuma.yaml
 - grafana.yaml
+- prometheus-project.yaml
+- prometheus.yaml
 - visual-studio-code.yaml
 - wiki-js.yaml
 - paperless-project.yaml
@@ -0,0 +1,55 @@
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: prometheus
+  namespace: argocd
+spec:
+  description: Prometheus monitoring stack and cluster-specific monitoring configuration
+  sourceRepos:
+    - https://prometheus-community.github.io/helm-charts
+    - https://code.brunner.ninja/feedc0de/homelab-gitops.git
+  destinations:
+    - namespace: prometheus
+      server: https://kubernetes.default.svc
+  clusterResourceWhitelist:
+    - group: apiextensions.k8s.io
+      kind: CustomResourceDefinition
+    - group: rbac.authorization.k8s.io
+      kind: ClusterRole
+    - group: rbac.authorization.k8s.io
+      kind: ClusterRoleBinding
+    - group: admissionregistration.k8s.io
+      kind: MutatingWebhookConfiguration
+    - group: admissionregistration.k8s.io
+      kind: ValidatingWebhookConfiguration
+  namespaceResourceWhitelist:
+    - group: ""
+      kind: ConfigMap
+    - group: ""
+      kind: Endpoints
+    - group: ""
+      kind: Secret
+    - group: ""
+      kind: Service
+    - group: ""
+      kind: ServiceAccount
+    - group: apps
+      kind: DaemonSet
+    - group: apps
+      kind: Deployment
+    - group: batch
+      kind: Job
+    - group: monitoring.coreos.com
+      kind: Alertmanager
+    - group: monitoring.coreos.com
+      kind: Prometheus
+    - group: monitoring.coreos.com
+      kind: PrometheusRule
+    - group: monitoring.coreos.com
+      kind: ServiceMonitor
+    - group: networking.k8s.io
+      kind: Ingress
+    - group: rbac.authorization.k8s.io
+      kind: Role
+    - group: rbac.authorization.k8s.io
+      kind: RoleBinding
@@ -0,0 +1,31 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: prometheus
+  namespace: argocd
+spec:
+  project: prometheus
+  sources:
+    - repoURL: https://prometheus-community.github.io/helm-charts
+      chart: kube-prometheus-stack
+      targetRevision: 92.10.1
+      helm:
+        releaseName: prometheus
+        valueFiles:
+          - $values/apps/prometheus/values.yaml
+    - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git
+      targetRevision: main
+      ref: values
+    - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git
+      targetRevision: main
+      path: apps/prometheus/resources
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: prometheus
+  syncPolicy:
+    automated:
+      enabled: true
+      prune: false
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=false
@@ -0,0 +1,21 @@
+# kubeadm control-plane metrics access
+
+This directory is the tracked source for arschrock's kubeadm control-plane
+metrics configuration. It is intentionally not an Argo CD Application: these
+files configure the host's kubeadm-generated static pods, systemd, and
+iptables, which Argo CD cannot reconcile on the host.
+
+Apply host changes only through the documented administrator procedure. Before
+regenerating static pods, take and verify an etcd snapshot and keep a copy off
+the control-plane host. Back up existing manifests outside
+`/etc/kubernetes/manifests`; kubelet treats every non-dot file there as a
+static-pod manifest.
+
+The controller-manager and scheduler metrics endpoints use authenticated HTTPS.
+Etcd's separate port 2381 is unauthenticated, so the systemd-managed firewall
+restricts access to localhost and the six Odroid node addresses. Prometheus is
+currently scheduled on an Odroid node, and Cilium masquerades pod-to-node
+connections to that node's address.
+
+When adding control-plane nodes, apply the same kubeadm component arguments and
+update the firewall allowlist for the worker node addresses.
@@ -0,0 +1,50 @@
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: 192.168.0.2
+  bindPort: 6443
+nodeRegistration:
+  criSocket: unix:///var/run/crio/crio.sock
+  name: arschrock
+  taints:
+    - effect: NoSchedule
+      key: node-role.kubernetes.io/control-plane
+timeouts:
+  controlPlaneComponentHealthCheck: 4m0s
+  discovery: 5m0s
+  etcdAPICall: 2m0s
+  kubeletHealthCheck: 4m0s
+  kubernetesAPICall: 1m0s
+  tlsBootstrap: 5m0s
+  upgradeManifests: 5m0s
+---
+apiServer: {}
+apiVersion: kubeadm.k8s.io/v1beta4
+caCertificateValidityPeriod: 87600h0m0s
+certificateValidityPeriod: 8760h0m0s
+certificatesDir: /etc/kubernetes/pki
+clusterName: kubernetes
+controllerManager:
+  extraArgs:
+    - name: bind-address
+      value: 0.0.0.0
+dns: {}
+encryptionAlgorithm: RSA-2048
+etcd:
+  local:
+    dataDir: /var/lib/etcd
+    extraArgs:
+      - name: listen-metrics-urls
+        value: http://0.0.0.0:2381
+imageRepository: registry.k8s.io
+kind: ClusterConfiguration
+kubernetesVersion: v1.36.3
+networking:
+  dnsDomain: cluster.local
+  serviceSubnet: 10.96.0.0/12
+proxy:
+  disabled: true
+scheduler:
+  extraArgs:
+    - name: bind-address
+      value: 0.0.0.0
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+set -eu
+
+chain=KUBEADM_METRICS
+
+remove_jump() {
+    while iptables -C INPUT -p tcp --dport 2381 -j "$chain" 2>/dev/null; do
+        iptables -D INPUT -p tcp --dport 2381 -j "$chain"
+    done
+}
+
+case "${1:-start}" in
+    start)
+        iptables -N "$chain" 2>/dev/null || true
+        iptables -F "$chain"
+
+        iptables -A "$chain" -s 127.0.0.0/8 -j ACCEPT
+        iptables -A "$chain" -s 192.168.0.157/32 -j ACCEPT
+        iptables -A "$chain" -s 192.168.0.141/32 -j ACCEPT
+        iptables -A "$chain" -s 192.168.0.74/32 -j ACCEPT
+        iptables -A "$chain" -s 192.168.0.58/32 -j ACCEPT
+        iptables -A "$chain" -s 192.168.0.147/32 -j ACCEPT
+        iptables -A "$chain" -s 192.168.0.60/32 -j ACCEPT
+        iptables -A "$chain" -p tcp -j REJECT --reject-with tcp-reset
+
+        remove_jump
+        iptables -I INPUT 1 -p tcp --dport 2381 -j "$chain"
+        ;;
+    stop)
+        remove_jump
+        iptables -F "$chain" 2>/dev/null || true
+        iptables -X "$chain" 2>/dev/null || true
+        ;;
+    *)
+        echo "Usage: $0 {start|stop}" >&2
+        exit 2
+        ;;
+esac
@@ -0,0 +1,14 @@
+[Unit]
+Description=Restrict unauthenticated kubeadm etcd metrics
+After=network-pre.target
+Before=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/kubernetes-metrics-firewall start
+ExecReload=/usr/local/sbin/kubernetes-metrics-firewall start
+ExecStop=/usr/local/sbin/kubernetes-metrics-firewall stop
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target