From bbabf768ecbc9c549df2a0f04733a60d838e4810 Mon Sep 17 00:00:00 2001
From: 0xFEEDC0DE64 <daniel@brunner.ninja>
Date: Sat, 31 Oct 2026 23:52:50 +0100
Subject: [PATCH] Right-size CPU requests and stage controlled Odroid
 rebalancing

---
 apps/authentik/values.yaml          |  5 ++-
 apps/brunner-ninja/values.yaml      |  4 ++
 apps/coturn/values.yaml             |  5 +++
 apps/descheduler/README.md          | 24 +++++++++++
 apps/descheduler/values.yaml        | 58 +++++++++++++++++++++++++
 apps/elastic/operator-values.yaml   |  4 +-
 apps/elastic/resources/netflow.yaml |  2 +-
 apps/elastic/resources/stack.yaml   |  5 +--
 apps/grafana/values.yaml            |  4 ++
 apps/home-assistant/values.yaml     |  4 ++
 apps/influxdb2/values.yaml          |  4 +-
 apps/mastodon/values.yaml           |  6 +--
 apps/mktxp/values.yaml              |  5 +++
 apps/paperless/values.yaml          |  2 +-
 apps/pdf-editor/values.yaml         |  2 +-
 apps/prometheus/values.yaml         |  3 ++
 apps/scanner/values.yaml            |  2 +-
 apps/traefik/values.yaml            |  2 +-
 apps/uptime-kuma/values.yaml        |  2 +-
 bootstrap/descheduler-project.yaml  | 29 +++++++++++++
 bootstrap/descheduler.yaml          | 26 ++++++++++++
 bootstrap/kustomization.yaml        |  2 +
 docs/cpu-sizing.md                  | 66 +++++++++++++++++++++++++++++
 23 files changed, 249 insertions(+), 17 deletions(-)
 create mode 100644 apps/descheduler/README.md
 create mode 100644 apps/descheduler/values.yaml
 create mode 100644 bootstrap/descheduler-project.yaml
 create mode 100644 bootstrap/descheduler.yaml
 create mode 100644 docs/cpu-sizing.md

diff --git a/apps/authentik/values.yaml b/apps/authentik/values.yaml
index 3b16dac..9434a68 100644
--- a/apps/authentik/values.yaml
+++ b/apps/authentik/values.yaml
@@ -49,7 +49,7 @@ server:
     minAvailable: 1
   resources:
     requests:
-      cpu: 500m
+      cpu: 150m
       memory: 1Gi
     limits:
       # Deliberately leave CPU unlimited so login bursts can use idle cores.
@@ -91,6 +91,9 @@ server:
   startupProbe:
     timeoutSeconds: 10
 worker:
+  resources:
+    requests:
+      cpu: 150m
   enabled: true
   livenessProbe:
     timeoutSeconds: 10
diff --git a/apps/brunner-ninja/values.yaml b/apps/brunner-ninja/values.yaml
index 08710f9..94c85a9 100644
--- a/apps/brunner-ninja/values.yaml
+++ b/apps/brunner-ninja/values.yaml
@@ -20,3 +20,7 @@ ingress:
   - hosts:
     - brunner.ninja
     secretName: brunner-ninja-tls
+
+resources:
+  requests:
+    cpu: 175m
diff --git a/apps/coturn/values.yaml b/apps/coturn/values.yaml
index 0b05da0..2fedb85 100644
--- a/apps/coturn/values.yaml
+++ b/apps/coturn/values.yaml
@@ -23,3 +23,8 @@ certificate:
   secretName: coturn
   dnsName: brunner.ninja
   issuerName: letsencrypt-dns
+resources:
+  requests:
+    cpu: 50m
+  limits:
+    cpu:
diff --git a/apps/descheduler/README.md b/apps/descheduler/README.md
new file mode 100644
index 0000000..2de512e
--- /dev/null
+++ b/apps/descheduler/README.md
@@ -0,0 +1,24 @@
+# Odroid CPU rebalancing
+
+The upstream Descheduler chart runs every five minutes. `values.yaml` is the
+policy; the bootstrap Application selects its chart version.
+
+The policy uses Metrics Server CPU usage: nodes below 40% are possible
+receivers and nodes above 75% are possible eviction sources. Kubernetes still
+chooses replacement placement. It does not live-migrate processes or guarantee
+that a replacement lands on a different node.
+
+Each cycle permits at most two evictions overall, one per node and namespace.
+Pods must be at least one hour old, belong to a controller with at least two
+replicas, have a PodDisruptionBudget, have no PVC, and fit another eligible
+node. Only the six Odroids are eligible. Storage, networking, monitoring,
+certificate management, and GitOps namespaces are excluded. EmptyDir alone is
+not a protection for these replicated stateless workloads.
+
+Start with `cmdOptions.dry-run: true`, inspect Job logs, then remove that option
+to allow evictions. To stop balancing, set `suspend: true` in values and let Argo
+sync. Existing running Jobs must finish before suspension takes full effect.
+
+Review after changes to workload mix; do not remove PVC and infrastructure
+protections to force a more even CPU graph. CPU requests determine whether a
+replacement can fit even when observed utilization is low.
diff --git a/apps/descheduler/values.yaml b/apps/descheduler/values.yaml
new file mode 100644
index 0000000..06399b7
--- /dev/null
+++ b/apps/descheduler/values.yaml
@@ -0,0 +1,58 @@
+schedule: "*/5 * * * *"
+concurrencyPolicy: Forbid
+activeDeadlineSeconds: 120
+successfulJobsHistoryLimit: 2
+failedJobsHistoryLimit: 2
+resources:
+  requests:
+    cpu: 50m
+    memory: 128Mi
+  limits:
+    cpu: null
+    memory: 256Mi
+# Observe the first cycle before enabling evictions.
+cmdOptions:
+  dry-run: true
+  v: 3
+deschedulerPolicy:
+  maxNoOfPodsToEvictPerNode: 1
+  maxNoOfPodsToEvictPerNamespace: 1
+  maxNoOfPodsToEvictTotal: 2
+  metricsProviders:
+    - source: KubernetesMetrics
+  profiles:
+    - name: odroid-cpu-balance
+      pluginConfig:
+        - name: DefaultEvictor
+          args:
+            nodeFit: true
+            minPodAge: 1h
+            minReplicas: 2
+            nodeSelector: "kubernetes.io/hostname in (odroid-h3-00,odroid-h3-01,odroid-h3-02,odroid-h3-03,odroid-h3-04,odroid-h3-05)"
+            podProtections:
+              defaultDisabled:
+                - PodsWithLocalStorage
+              extraEnabled:
+                - PodsWithPVC
+                - PodsWithoutPDB
+        - name: LowNodeUtilization
+          args:
+            metricsUtilization:
+              source: KubernetesMetrics
+            thresholds:
+              cpu: 40
+            targetThresholds:
+              cpu: 75
+            evictableNamespaces:
+              exclude:
+                - kube-system
+                - rook-ceph
+                - argocd
+                - elastic
+                - prometheus
+                - metallb-system
+                - cert-manager
+      plugins:
+        balance:
+          enabled:
+            - LowNodeUtilization
diff --git a/apps/elastic/operator-values.yaml b/apps/elastic/operator-values.yaml
index 3159695..49dfb51 100644
--- a/apps/elastic/operator-values.yaml
+++ b/apps/elastic/operator-values.yaml
@@ -1,5 +1,5 @@
 managedNamespaces:
-  - elastic
+- elastic
 
 installCRDs: true
 createClusterScopedResources: true
@@ -9,7 +9,7 @@ resources:
     cpu: 100m
     memory: 256Mi
   limits:
-    cpu: 1
+    cpu: null
     memory: 1Gi
 
 config:
diff --git a/apps/elastic/resources/netflow.yaml b/apps/elastic/resources/netflow.yaml
index c0289a4..b1eb73d 100644
--- a/apps/elastic/resources/netflow.yaml
+++ b/apps/elastic/resources/netflow.yaml
@@ -98,7 +98,7 @@ spec:
                 protocol: UDP
             resources:
               requests:
-                cpu: 500m
+                cpu: 75m
                 memory: 1Gi
               limits:
                 cpu: "2"
diff --git a/apps/elastic/resources/stack.yaml b/apps/elastic/resources/stack.yaml
index e6c188f..9e3cbc1 100644
--- a/apps/elastic/resources/stack.yaml
+++ b/apps/elastic/resources/stack.yaml
@@ -47,10 +47,9 @@ spec:
             - name: elasticsearch
               resources:
                 requests:
-                  cpu: "2"
+                  cpu: 500m
                   memory: 12Gi
                 limits:
-                  cpu: "4"
                   memory: 12Gi
       volumeClaimTemplates:
         - metadata:
@@ -119,7 +118,7 @@ spec:
         - name: kibana
           resources:
             requests:
-              cpu: 500m
+              cpu: 100m
               memory: 4Gi
             limits:
               cpu: "2"
diff --git a/apps/grafana/values.yaml b/apps/grafana/values.yaml
index 76ec919..57c246a 100644
--- a/apps/grafana/values.yaml
+++ b/apps/grafana/values.yaml
@@ -82,3 +82,7 @@ grafana.ini:
   security:
     allow_embedding: true
     x_frame_options: ""
+
+resources:
+  requests:
+    cpu: 75m
diff --git a/apps/home-assistant/values.yaml b/apps/home-assistant/values.yaml
index 0b39d04..b22ba0a 100644
--- a/apps/home-assistant/values.yaml
+++ b/apps/home-assistant/values.yaml
@@ -20,3 +20,7 @@ ingress:
     annotations:
       traefik.ingress.kubernetes.io/router.entrypoints: websecure
       traefik.ingress.kubernetes.io/router.middlewares: default-authentik@kubernetescrd
+
+resources:
+  requests:
+    cpu: 175m
diff --git a/apps/influxdb2/values.yaml b/apps/influxdb2/values.yaml
index 6237207..8b0c8de 100644
--- a/apps/influxdb2/values.yaml
+++ b/apps/influxdb2/values.yaml
@@ -15,10 +15,10 @@ env:
   value: Europe/Vienna
 resources:
   requests:
-    cpu: 500m
+    cpu: 100m
     memory: 2Gi
   limits:
-    cpu: "2"
+    cpu:
     memory: 16Gi
 ingress:
   enabled: true
diff --git a/apps/mastodon/values.yaml b/apps/mastodon/values.yaml
index d6a2400..72a650c 100644
--- a/apps/mastodon/values.yaml
+++ b/apps/mastodon/values.yaml
@@ -49,7 +49,7 @@ mastodon:
     replicas: 1
     resources:
       requests:
-        cpu: 250m
+        cpu: 50m
         memory: 768Mi
       limits:
         cpu: "2"
@@ -70,7 +70,7 @@ mastodon:
           - fasp
         resources:
           requests:
-            cpu: 250m
+            cpu: 50m
             memory: 768Mi
           limits:
             cpu: "2"
@@ -83,7 +83,7 @@ mastodon:
     workers: 1
     resources:
       requests:
-        cpu: 100m
+        cpu: 25m
         memory: 256Mi
       limits:
         cpu: "1"
diff --git a/apps/mktxp/values.yaml b/apps/mktxp/values.yaml
index 66811a1..11bc926 100644
--- a/apps/mktxp/values.yaml
+++ b/apps/mktxp/values.yaml
@@ -4,3 +4,8 @@ serviceMonitor:
   enabled: true
   labels:
     release: prometheus
+resources:
+  requests:
+    cpu: 175m
+  limits:
+    cpu:
diff --git a/apps/paperless/values.yaml b/apps/paperless/values.yaml
index 283641f..63f7d11 100644
--- a/apps/paperless/values.yaml
+++ b/apps/paperless/values.yaml
@@ -44,5 +44,5 @@ affinity:
               values: [arschrock]
 resources:
   requests:
-    cpu: 100m
+    cpu: 50m
     memory: 1Gi
diff --git a/apps/pdf-editor/values.yaml b/apps/pdf-editor/values.yaml
index 0d512db..221c61e 100644
--- a/apps/pdf-editor/values.yaml
+++ b/apps/pdf-editor/values.yaml
@@ -7,7 +7,7 @@ ingress:
     traefik.ingress.kubernetes.io/router.middlewares: default-authentik@kubernetescrd
 resources:
   requests:
-    cpu: 100m
+    cpu: 35m
     memory: 512Mi
   limits:
     memory: 2Gi
diff --git a/apps/prometheus/values.yaml b/apps/prometheus/values.yaml
index d1d110a..df9e0aa 100644
--- a/apps/prometheus/values.yaml
+++ b/apps/prometheus/values.yaml
@@ -77,6 +77,9 @@ prometheus:
       kubernetes.io/ingress.class: "traefik"
       traefik.ingress.kubernetes.io/router.entrypoints: websecure
   prometheusSpec:
+    resources:
+      requests:
+        cpu: 200m
     retention: "15d"
     storageSpec:
       volumeClaimTemplate:
diff --git a/apps/scanner/values.yaml b/apps/scanner/values.yaml
index d48a76d..96b1b63 100644
--- a/apps/scanner/values.yaml
+++ b/apps/scanner/values.yaml
@@ -13,5 +13,5 @@ ingress:
     traefik.ingress.kubernetes.io/router.middlewares: default-authentik@kubernetescrd
 resources:
   requests:
-    cpu: 100m
+    cpu: 25m
     memory: 256Mi
diff --git a/apps/traefik/values.yaml b/apps/traefik/values.yaml
index 7db65f1..1b9c6f3 100644
--- a/apps/traefik/values.yaml
+++ b/apps/traefik/values.yaml
@@ -11,7 +11,7 @@ podDisruptionBudget:
   minAvailable: 1
 resources:
   requests:
-    cpu: 500m
+    cpu: 700m
     memory: 1Gi
 affinity:
   podAntiAffinity:
diff --git a/apps/uptime-kuma/values.yaml b/apps/uptime-kuma/values.yaml
index 6018b51..82fddf1 100644
--- a/apps/uptime-kuma/values.yaml
+++ b/apps/uptime-kuma/values.yaml
@@ -20,7 +20,7 @@ readinessProbe:
   timeoutSeconds: 5
 resources:
   requests:
-    cpu: 100m
+    cpu: 75m
     memory: 256Mi
     ephemeral-storage: 128Mi
   limits:
diff --git a/bootstrap/descheduler-project.yaml b/bootstrap/descheduler-project.yaml
new file mode 100644
index 0000000..54abe2a
--- /dev/null
+++ b/bootstrap/descheduler-project.yaml
@@ -0,0 +1,29 @@
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: descheduler
+  namespace: argocd
+spec:
+  description: Controlled CPU rebalancing on the Odroid workers
+  sourceRepos:
+    - https://kubernetes-sigs.github.io/descheduler/
+    - https://code.brunner.ninja/feedc0de/homelab-gitops.git
+  destinations:
+    - namespace: kube-system
+      server: https://kubernetes.default.svc
+  clusterResourceWhitelist:
+    - group: rbac.authorization.k8s.io
+      kind: ClusterRole
+    - group: rbac.authorization.k8s.io
+      kind: ClusterRoleBinding
+  namespaceResourceWhitelist:
+    - group: ""
+      kind: ConfigMap
+    - group: ""
+      kind: ServiceAccount
+    - group: batch
+      kind: CronJob
+    - group: rbac.authorization.k8s.io
+      kind: Role
+    - group: rbac.authorization.k8s.io
+      kind: RoleBinding
diff --git a/bootstrap/descheduler.yaml b/bootstrap/descheduler.yaml
new file mode 100644
index 0000000..466a57c
--- /dev/null
+++ b/bootstrap/descheduler.yaml
@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: descheduler
+  namespace: argocd
+spec:
+  project: descheduler
+  sources:
+    - repoURL: https://kubernetes-sigs.github.io/descheduler/
+      chart: descheduler
+      targetRevision: 0.36.0
+      helm:
+        releaseName: descheduler
+        valueFiles:
+          - $values/apps/descheduler/values.yaml
+    - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      enabled: true
+      prune: true
+      selfHeal: true
diff --git a/bootstrap/kustomization.yaml b/bootstrap/kustomization.yaml
index 91843ee..8ef41cc 100644
--- a/bootstrap/kustomization.yaml
+++ b/bootstrap/kustomization.yaml
@@ -151,3 +151,5 @@ resources:
 - ghostfolio.yaml
 - oldipmi-project.yaml
 - oldipmi.yaml
+- descheduler-project.yaml
+- descheduler.yaml
diff --git a/docs/cpu-sizing.md b/docs/cpu-sizing.md
new file mode 100644
index 0000000..ad2228a
--- /dev/null
+++ b/docs/cpu-sizing.md
@@ -0,0 +1,66 @@
+# CPU sizing and balancing
+
+## Review on 2026-10-31
+
+Prometheus was queried for the 95th percentile of five-minute CPU rates over
+its available history within the previous 14 days, sampled every 15 minutes:
+
+```promql
+quantile_over_time(0.95,
+  (sum by(namespace,pod,container)
+    (rate(container_cpu_usage_seconds_total{container!="",container!="POD"}[5m]))
+  )[14d:15m])
+```
+
+The largest observed replica value was considered for shared workload sizing.
+New pods have shorter histories; this is not a guarantee of fourteen complete
+days per workload. These measurements include the recent recovery period.
+
+The starting recommendation is p95 times 1.2, rounded upwards. Request floors
+are retained for startup, databases, recovery, and bursty workloads. Requests
+are not CPU limits. Memory requests and limits are unchanged.
+
+Reviewed changes include Elasticsearch/Kibana, Netflow, Authentik, Traefik,
+InfluxDB, MKTXP, Mastodon, Paperless, scanner/PDF editor, Uptime Kuma, TURN,
+Home Assistant, WordPress, Grafana and Prometheus. Busy services without CPU
+requests receive explicit reservations. Elasticsearch retains a 500m floor
+per data node for startup/recovery, substantially above its measured baseline.
+
+Matrix is still a legacy Helm release. Its canonical values live in
+`matrix/authentication/values.yaml` in the homelab workspace: the main Synapse
+process gets 500m, federation sender 225m, and other workers 50m. Init-container
+requests can make effective pod reservations higher than the main container.
+The shared RBD claim and worker/offload affinity remain unchanged.
+
+Ceph daemons, the API server, Cilium, CoreDNS, Argo CD, and ingress already have
+no CPU limits. Their low or absent requests are not interpreted as spare
+physical capacity; Descheduler uses measured CPU. The legacy Ceph CSI operator
+CPU cap is removed in `rook/deploy/examples/csi-operator.yaml`. Ceph daemon
+resources are left alone to avoid a storage-wide restart. CPU caps are also
+removed from the Elastic operator/data nodes, InfluxDB, MKTXP, and TURN.
+
+## Deliberate exclusions
+
+- Do not infer batch, transcoding, game-server or GPU requirements from idle
+  CPU measurements. Their explicit reservations are retained.
+- Do not resize workloads that are down or lack representative history.
+- Do not automatically rewrite operator-managed database resources or apply
+  VPA mutation alongside Argo. This review changes owning configuration only.
+- Loops and Pixelfed charts hard-code some resource requests; overriding an
+  unsupported values key would do nothing. Those require a separate chart
+  interface change before their requests can be tuned through GitOps.
+- Small existing requests are retained where a reduction offers little useful
+  headroom. The aim is schedulability and contention protection, not identical
+  requests across applications.
+
+## Ongoing operation
+
+Descheduler runs automatically; request changes are reviewed configuration,
+not a continuously mutating autoscaler. Repeat the history review after a
+representative week or a substantial workload change. Never set every request
+from a single current-usage sample. Monitor pending pods, CPU throttling,
+latency, and node CPU usage as well as reserved capacity.
+
+For a new request change, render the owning chart, inspect all container/init
+resources, and reconcile through Argo (or the existing legacy Helm installer).
+Keep the prior commit as rollback; do not delete PVCs to resolve placement.