diff --git a/apps/prometheus/README.md b/apps/prometheus/README.md new file mode 100644 index 0000000..c3e856a --- /dev/null +++ b/apps/prometheus/README.md @@ -0,0 +1,19 @@ +# Prometheus + +Argo CD installs the upstream `kube-prometheus-stack` Helm chart at the +version selected in `bootstrap/prometheus.yaml`. `values.yaml` contains only +this cluster's intentional settings. The chart owns Prometheus, Alertmanager, +their supporting workloads, and chart-provided monitoring resources. + +The `resources/` directory contains cluster-specific scrape endpoints and +alert rules that are not chart defaults. It does not contain application +workload templates. + +Prometheus TSDB uses the existing 50Gi `rook-ceph-block` claim. During adoption, +preserve claim +`prometheus-prometheus-kube-prometheus-prometheus-db-prometheus-prometheus-kube-prometheus-prometheus-0` +and PV `pvc-dfeec228-10e1-42f2-af69-ebcac00f5ddc`; do not delete or recreate it. + +`kubeadm` control-plane metrics listener and host firewall settings are tracked +separately under `cluster-config/kubeadm-monitoring/`. Argo CD does not manage +those host files or static-pod manifests. diff --git a/apps/prometheus/resources/externals.yaml b/apps/prometheus/resources/externals.yaml new file mode 100644 index 0000000..2145b43 --- /dev/null +++ b/apps/prometheus/resources/externals.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Service +metadata: + name: external-immich-8082 + namespace: prometheus + labels: + app: external-immich-service + port: "8082" +spec: + ports: + - name: metrics + port: 8082 +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: external-immich-8082 + namespace: prometheus +subsets: + - addresses: + - ip: 192.168.0.2 + ports: + - name: metrics + port: 8082 +--- +apiVersion: v1 +kind: Service +metadata: + name: external-immich-8083 + namespace: prometheus + labels: + app: external-immich-service + port: "8083" +spec: + ports: + - name: metrics + port: 8083 +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: external-immich-8083 + namespace: prometheus +subsets: + - addresses: + - ip: 192.168.0.2 + ports: + - name: metrics + port: 8083 +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: external-immich-service + namespace: prometheus +spec: + selector: + matchLabels: + app: external-immich-service + endpoints: + - port: metrics + path: /metrics + interval: 30s diff --git a/apps/prometheus/resources/mikrotik-storage-alerts.yaml b/apps/prometheus/resources/mikrotik-storage-alerts.yaml new file mode 100644 index 0000000..962dddd --- /dev/null +++ b/apps/prometheus/resources/mikrotik-storage-alerts.yaml @@ -0,0 +1,29 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: mikrotik-storage + namespace: prometheus + labels: + release: prometheus +spec: + groups: + - name: mikrotik-storage + rules: + - alert: MikroTikStorageLow + expr: |- + mktxp_system_free_hdd_space < 131072 + and mktxp_system_free_hdd_space >= 65536 + for: 15m + labels: + severity: warning + annotations: + summary: "MikroTik flash is running low on {{ $labels.routerboard_name }}" + description: "{{ $labels.routerboard_name }} ({{ $labels.board_name }}) has only {{ $value | humanize1024 }}B of writable flash free. Configuration changes and upgrades may soon fail." + - alert: MikroTikStorageCritical + expr: mktxp_system_free_hdd_space < 65536 + for: 5m + labels: + severity: critical + annotations: + summary: "MikroTik flash is critically low on {{ $labels.routerboard_name }}" + description: "{{ $labels.routerboard_name }} ({{ $labels.board_name }}) has only {{ $value | humanize1024 }}B of writable flash free. Configuration changes and upgrades are at immediate risk." diff --git a/apps/prometheus/resources/rbd-volume-alerts.yaml b/apps/prometheus/resources/rbd-volume-alerts.yaml new file mode 100644 index 0000000..448f7da --- /dev/null +++ b/apps/prometheus/resources/rbd-volume-alerts.yaml @@ -0,0 +1,41 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: ceph-rbd-volume-capacity + namespace: prometheus + labels: + release: prometheus +spec: + groups: + - name: ceph-rbd-volume-capacity + rules: + - alert: CephRBDPersistentVolumeFillingUp + expr: |- + ( + 1 - kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"} + / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics"} + ) + * on(namespace,persistentvolumeclaim) group_left(storageclass,volumename) + kube_persistentvolumeclaim_info{storageclass=~"rook-ceph-block.*"} + > 0.80 + for: 10m + labels: + severity: warning + annotations: + summary: "Ceph RBD PVC is more than 80% full" + description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is {{ $value | humanizePercentage }} used." + - alert: CephRBDPersistentVolumeFillingUp + expr: |- + ( + 1 - kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"} + / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics"} + ) + * on(namespace,persistentvolumeclaim) group_left(storageclass,volumename) + kube_persistentvolumeclaim_info{storageclass=~"rook-ceph-block.*"} + > 0.90 + for: 5m + labels: + severity: critical + annotations: + summary: "Ceph RBD PVC is more than 90% full" + description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is {{ $value | humanizePercentage }} used." diff --git a/apps/prometheus/values.yaml b/apps/prometheus/values.yaml new file mode 100644 index 0000000..d1d110a --- /dev/null +++ b/apps/prometheus/values.yaml @@ -0,0 +1,92 @@ +grafana: + enabled: false + +defaultRules: + rules: + kubeProxy: false + +alertmanager: + ingress: + enabled: true + ingressClassName: traefik + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.middlewares: default-authentik@kubernetescrd + hosts: + - alertmanager.brunner.ninja + paths: + - / + pathType: ImplementationSpecific + + alertmanagerSpec: + externalUrl: https://alertmanager.brunner.ninja + routePrefix: / + + config: + global: + resolve_timeout: 5m + smtp_smarthost: smtp-relay.gmail.com:587 + smtp_from: Cluster Alerts + smtp_hello: brunner.ninja + smtp_require_tls: true + route: + group_by: + - namespace + - alertname + group_wait: 30s + group_interval: 5m + repeat_interval: 6h + receiver: cluster-email + routes: + - receiver: "null" + matchers: + - alertname =~ "Watchdog|InfoInhibitor" + - receiver: "null" + matchers: + - severity = "info" + - receiver: cluster-email + matchers: + - severity = "critical" + repeat_interval: 2h + continue: true + - receiver: cluster-email + matchers: + - severity != "critical" + continue: true + - receiver: cluster-matrix + receivers: + - name: "null" + - name: cluster-email + email_configs: + - to: Daniel Brunner + send_resolved: true + headers: + Subject: >- + [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}{{ if .CommonLabels.namespace }} in {{ .CommonLabels.namespace }}{{ end }} + - name: cluster-matrix + webhook_configs: + - url: http://matrix-alertmanager-receiver.prometheus.svc.cluster.local:12345/alerts/cluster + send_resolved: true + +prometheus: + ingress: + enabled: true + hosts: + - prometheus.brunner.ninja + annotations: + kubernetes.io/ingress.class: "traefik" + traefik.ingress.kubernetes.io/router.entrypoints: websecure + prometheusSpec: + retention: "15d" + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: rook-ceph-block + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi + serviceMonitorSelectorNilUsesHelmValues: false + serviceMonitorSelector: {} + ruleSelectorNilUsesHelmValues: false + ruleSelector: {} diff --git a/bootstrap/kustomization.yaml b/bootstrap/kustomization.yaml index 897aed4..61d15ad 100644 --- a/bootstrap/kustomization.yaml +++ b/bootstrap/kustomization.yaml @@ -60,6 +60,8 @@ resources: - tautulli.yaml - uptime-kuma.yaml - grafana.yaml +- prometheus-project.yaml +- prometheus.yaml - visual-studio-code.yaml - wiki-js.yaml - paperless-project.yaml diff --git a/bootstrap/prometheus-project.yaml b/bootstrap/prometheus-project.yaml new file mode 100644 index 0000000..e177530 --- /dev/null +++ b/bootstrap/prometheus-project.yaml @@ -0,0 +1,55 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: prometheus + namespace: argocd +spec: + description: Prometheus monitoring stack and cluster-specific monitoring configuration + sourceRepos: + - https://prometheus-community.github.io/helm-charts + - https://code.brunner.ninja/feedc0de/homelab-gitops.git + destinations: + - namespace: prometheus + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + namespaceResourceWhitelist: + - group: "" + kind: ConfigMap + - group: "" + kind: Endpoints + - group: "" + kind: Secret + - group: "" + kind: Service + - group: "" + kind: ServiceAccount + - group: apps + kind: DaemonSet + - group: apps + kind: Deployment + - group: batch + kind: Job + - group: monitoring.coreos.com + kind: Alertmanager + - group: monitoring.coreos.com + kind: Prometheus + - group: monitoring.coreos.com + kind: PrometheusRule + - group: monitoring.coreos.com + kind: ServiceMonitor + - group: networking.k8s.io + kind: Ingress + - group: rbac.authorization.k8s.io + kind: Role + - group: rbac.authorization.k8s.io + kind: RoleBinding diff --git a/bootstrap/prometheus.yaml b/bootstrap/prometheus.yaml new file mode 100644 index 0000000..48035ac --- /dev/null +++ b/bootstrap/prometheus.yaml @@ -0,0 +1,31 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: prometheus + namespace: argocd +spec: + project: prometheus + sources: + - repoURL: https://prometheus-community.github.io/helm-charts + chart: kube-prometheus-stack + targetRevision: 92.10.1 + helm: + releaseName: prometheus + valueFiles: + - $values/apps/prometheus/values.yaml + - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git + targetRevision: main + ref: values + - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git + targetRevision: main + path: apps/prometheus/resources + destination: + server: https://kubernetes.default.svc + namespace: prometheus + syncPolicy: + automated: + enabled: true + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=false diff --git a/cluster-config/kubeadm-monitoring/README.md b/cluster-config/kubeadm-monitoring/README.md new file mode 100644 index 0000000..a49e45c --- /dev/null +++ b/cluster-config/kubeadm-monitoring/README.md @@ -0,0 +1,21 @@ +# kubeadm control-plane metrics access + +This directory is the tracked source for arschrock's kubeadm control-plane +metrics configuration. It is intentionally not an Argo CD Application: these +files configure the host's kubeadm-generated static pods, systemd, and +iptables, which Argo CD cannot reconcile on the host. + +Apply host changes only through the documented administrator procedure. Before +regenerating static pods, take and verify an etcd snapshot and keep a copy off +the control-plane host. Back up existing manifests outside +`/etc/kubernetes/manifests`; kubelet treats every non-dot file there as a +static-pod manifest. + +The controller-manager and scheduler metrics endpoints use authenticated HTTPS. +Etcd's separate port 2381 is unauthenticated, so the systemd-managed firewall +restricts access to localhost and the six Odroid node addresses. Prometheus is +currently scheduled on an Odroid node, and Cilium masquerades pod-to-node +connections to that node's address. + +When adding control-plane nodes, apply the same kubeadm component arguments and +update the firewall allowlist for the worker node addresses. diff --git a/cluster-config/kubeadm-monitoring/kubeadm-config.yaml b/cluster-config/kubeadm-monitoring/kubeadm-config.yaml new file mode 100644 index 0000000..aba632e --- /dev/null +++ b/cluster-config/kubeadm-monitoring/kubeadm-config.yaml @@ -0,0 +1,50 @@ +apiVersion: kubeadm.k8s.io/v1beta4 +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: 192.168.0.2 + bindPort: 6443 +nodeRegistration: + criSocket: unix:///var/run/crio/crio.sock + name: arschrock + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane +timeouts: + controlPlaneComponentHealthCheck: 4m0s + discovery: 5m0s + etcdAPICall: 2m0s + kubeletHealthCheck: 4m0s + kubernetesAPICall: 1m0s + tlsBootstrap: 5m0s + upgradeManifests: 5m0s +--- +apiServer: {} +apiVersion: kubeadm.k8s.io/v1beta4 +caCertificateValidityPeriod: 87600h0m0s +certificateValidityPeriod: 8760h0m0s +certificatesDir: /etc/kubernetes/pki +clusterName: kubernetes +controllerManager: + extraArgs: + - name: bind-address + value: 0.0.0.0 +dns: {} +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + - name: listen-metrics-urls + value: http://0.0.0.0:2381 +imageRepository: registry.k8s.io +kind: ClusterConfiguration +kubernetesVersion: v1.36.3 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + - name: bind-address + value: 0.0.0.0 diff --git a/cluster-config/kubeadm-monitoring/kubernetes-metrics-firewall b/cluster-config/kubeadm-monitoring/kubernetes-metrics-firewall new file mode 100755 index 0000000..a50a05d --- /dev/null +++ b/cluster-config/kubeadm-monitoring/kubernetes-metrics-firewall @@ -0,0 +1,39 @@ +#!/bin/sh + +set -eu + +chain=KUBEADM_METRICS + +remove_jump() { + while iptables -C INPUT -p tcp --dport 2381 -j "$chain" 2>/dev/null; do + iptables -D INPUT -p tcp --dport 2381 -j "$chain" + done +} + +case "${1:-start}" in + start) + iptables -N "$chain" 2>/dev/null || true + iptables -F "$chain" + + iptables -A "$chain" -s 127.0.0.0/8 -j ACCEPT + iptables -A "$chain" -s 192.168.0.157/32 -j ACCEPT + iptables -A "$chain" -s 192.168.0.141/32 -j ACCEPT + iptables -A "$chain" -s 192.168.0.74/32 -j ACCEPT + iptables -A "$chain" -s 192.168.0.58/32 -j ACCEPT + iptables -A "$chain" -s 192.168.0.147/32 -j ACCEPT + iptables -A "$chain" -s 192.168.0.60/32 -j ACCEPT + iptables -A "$chain" -p tcp -j REJECT --reject-with tcp-reset + + remove_jump + iptables -I INPUT 1 -p tcp --dport 2381 -j "$chain" + ;; + stop) + remove_jump + iptables -F "$chain" 2>/dev/null || true + iptables -X "$chain" 2>/dev/null || true + ;; + *) + echo "Usage: $0 {start|stop}" >&2 + exit 2 + ;; +esac diff --git a/cluster-config/kubeadm-monitoring/kubernetes-metrics-firewall.service b/cluster-config/kubeadm-monitoring/kubernetes-metrics-firewall.service new file mode 100644 index 0000000..a614a1e --- /dev/null +++ b/cluster-config/kubeadm-monitoring/kubernetes-metrics-firewall.service @@ -0,0 +1,14 @@ +[Unit] +Description=Restrict unauthenticated kubeadm etcd metrics +After=network-pre.target +Before=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/kubernetes-metrics-firewall start +ExecReload=/usr/local/sbin/kubernetes-metrics-firewall start +ExecStop=/usr/local/sbin/kubernetes-metrics-firewall stop +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target