Fix alert recovery, bound backup job history and expand data volumes

apps/backup/README.md

@@ -24,6 +24,6 @@ The RGW job syncs all 12 current buckets with `rclone sync`. It reads Rook-gener
 
 ## Schedules and alerts
 
-PostgreSQL runs daily between 02:15 and 06:30 Europe/Vienna; CephFS mirrors run between 07:00 and 12:30; RGW starts at 13:00. CronJobs forbid overlapping runs of the same backup. Prometheus alerts on failed backup Jobs, daily backup CronJobs without a success in 25 hours, and the S3 mirror without a success in 48 hours. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory.
+CronJob manifests define the schedules in Europe/Vienna and forbid overlapping runs of the same backup. Prometheus alerts on failed backup Jobs, daily backup CronJobs without a success in 25 hours, and the S3 mirror without a success in 48 hours. Finished Job records and pod logs expire after one day; backup files are retained. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory.
 
 Valkey/Redis persistence is not included yet. etcd and RBD remain deferred. Existing files and old scripts directly under `/mnt/backup` are preserved. No restore has been performed; the PostgreSQL archive checks validate archive readability, not a full restore rehearsal.

apps/backup/arschrock-postgresql.yaml

@@ -47,6 +47,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 0
+      ttlSecondsAfterFinished: 86400
       activeDeadlineSeconds: 14400
       template:
         spec:

apps/backup/cephfs/base/resources.yaml

@@ -29,6 +29,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 0
+      ttlSecondsAfterFinished: 86400
       activeDeadlineSeconds: 21600
       template:
         spec:

apps/backup/postgresql/base/resources.yaml

@@ -53,6 +53,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 0
+      ttlSecondsAfterFinished: 86400
       activeDeadlineSeconds: 21600
       template:
         spec:

apps/backup/s3/resources.yaml

@@ -62,6 +62,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 0
+      ttlSecondsAfterFinished: 86400
       activeDeadlineSeconds: 172800
       template:
         spec:

apps/elastic/resources/stack.yaml

@@ -60,7 +60,7 @@ spec:
             storageClassName: rook-ceph-block-ec
             resources:
               requests:
-                storage: 60Gi
+                storage: 80Gi
   http:
     service:
       spec:

apps/ollama/resources/models-pvc.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ollama-models
+  namespace: ollama
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false,Delete=false
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: rook-ceph-block-ec
+  volumeMode: Filesystem
+  volumeName: pvc-85f06c23-8e53-461d-94bc-b7dce0466883
+  resources:
+    requests:
+      storage: 500Gi

apps/prometheus/resources/backup-alerts.yaml

@@ -26,7 +26,7 @@ spec:
             or
             (
               (kube_cronjob_info{cronjob=~".*backup.*",cronjob!="s3-backup"}
-                unless kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
+                unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
               and on(namespace,cronjob)
               (time() - kube_cronjob_created{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000)
             )
@@ -42,7 +42,7 @@ spec:
             or
             (
               (kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup"}
-                unless kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
+                unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
               and on(namespace,cronjob)
               (time() - kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"} > 172800)
             )

apps/prometheus/resources/cpu-capacity-alerts.yaml

@@ -0,0 +1,41 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cpu-capacity
+  namespace: prometheus
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: cpu-capacity
+      rules:
+        - alert: KubeCPUOvercommit
+          # The upstream HA branch also fires on single-control-plane clusters.
+          # Only require largest-node failover capacity when the control plane is HA.
+          expr: |-
+            (
+              (
+                sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum)
+                - sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+              ) > 0
+              and on (cluster)
+              count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) < 3
+            )
+            or
+            (
+              (
+                sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum)
+                - (
+                  sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+                  - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+                )
+              ) > 0
+              and on (cluster)
+              count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) >= 3
+            )
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: Cluster has overcommitted CPU resource requests.
+            description: "CPU requests exceed usable capacity by {{ $value }} cores (total capacity for a non-HA control plane; largest-node failure capacity for HA)."

apps/prometheus/values.yaml

@@ -2,9 +2,20 @@ grafana:
   enabled: false
 
 defaultRules:
+  disabled:
+    KubeCPUOvercommit: true
   rules:
     kubeProxy: false
 
+prometheus-node-exporter:
+  prometheus:
+    monitor:
+      metricRelabelings:
+        # These bind mounts share /mnt's capacity; retain its canonical gauges.
+        - sourceLabels: [__name__, fstype, mountpoint]
+          regex: 'node_filesystem_.+;bcachefs;(/komposthaufen|/var/www|/var/lib/(lidarr|prowlarr|radarr|readarr|sonarr|whisparr)|/home/feedc0de/(\.steam/steamapps|mcserver/plugins/dynmap/web/tiles))'
+          action: drop
+
 alertmanager:
   ingress:
     enabled: true

bootstrap/host-config/journald-retention.conf

@@ -0,0 +1,4 @@
+[Journal]
+SystemMaxUse=256M
+SystemKeepFree=2G
+MaxRetentionSec=7day

bootstrap/ollama.yaml

@@ -18,6 +18,9 @@ spec:
     - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git
       targetRevision: main
       ref: values
+    - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git
+      targetRevision: main
+      path: apps/ollama/resources
   destination:
     server: https://kubernetes.default.svc
     namespace: ollama

docs/alert-recovery.md

@@ -0,0 +1,27 @@
+# Alert recovery
+
+Backup success is the CronJob's `lastSuccessfulTime`, not the continued
+existence of an older failed Job. Backup rules must match CronJob info and
+success metrics explicitly on `(namespace,cronjob)`; info has extra schedule
+labels. Test both recovery and real failures with
+`python3 tests/run-alert-recovery.py` (PyYAML and promtool or Podman required).
+
+Backup Job metadata and pod logs expire one day after completion or failure.
+This does not delete backup files. Stale-backup rules continue to alert when
+successful backups are missing, even after failed Job records expire.
+During recovery, retire old failed Jobs only after confirming a newer success
+for their owning CronJob. Do not mark a backup successful manually.
+
+The CPU-capacity alert preserves the upstream total-capacity check for a
+non-HA control plane and applies largest-node failure capacity only to HA
+control planes. Requests remain based on the review in `cpu-sizing.md`.
+This does not promise failover for workloads pinned to arschrock.
+
+On odroid-h3-04, persistent journald storage is bounded using
+`bootstrap/host-config/journald-retention.conf`, installed as
+`/etc/systemd/journald.conf.d/homelab-retention.conf`. This host configuration
+requires installation through SSH; Argo does not manage host systemd files.
+The node's small system disk also holds active container images, so deleting
+unused images alone may free nothing. Never remove Ceph monitor database files
+to make space. Ceph crash reports may be archived after reviewing the cause
+and verifying it has stopped; archival retains the reports for investigation.

tests/alert-recovery.test.yaml

@@ -0,0 +1,84 @@
+# Rule files are generated from the PrometheusRule manifests by the test runner.
+rule_files:
+  - backup.rules.yaml
+  - cpu.rules.yaml
+evaluation_interval: 1m
+tests:
+  - name: Successful backups match CronJob info despite its extra schedule labels
+    interval: 1m
+    input_series:
+      - series: 'kube_cronjob_info{namespace="test",cronjob="postgresql-backup",schedule="0 2 * * *",concurrency_policy="Forbid"}'
+        values: '1+0x4320'
+      - series: 'kube_cronjob_created{namespace="test",cronjob="postgresql-backup"}'
+        values: '0+0x4320'
+      - series: 'kube_cronjob_status_last_successful_time{namespace="test",cronjob="postgresql-backup"}'
+        values: '255600+0x4320'
+      - series: 'kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup",schedule="0 16 * * *",concurrency_policy="Forbid"}'
+        values: '1+0x4320'
+      - series: 'kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"}'
+        values: '0+0x4320'
+      - series: 'kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"}'
+        values: '255600+0x4320'
+    promql_expr_test:
+      - expr: 'count(ALERTS{alertstate="firing",alertname=~"ScheduledBackupStale|S3BackupStale"}) or vector(0)'
+        eval_time: 72h
+        exp_samples:
+          - labels: '{}'
+            value: 0
+  - name: Stale and never successful backups still alert
+    interval: 1m
+    input_series:
+      - series: 'kube_cronjob_info{namespace="test",cronjob="never-backup",schedule="0 2 * * *"}'
+        values: '1+0x1560'
+      - series: 'kube_cronjob_created{namespace="test",cronjob="never-backup"}'
+        values: '0+0x1560'
+      - series: 'kube_cronjob_status_last_successful_time{namespace="test",cronjob="stale-backup"}'
+        values: '0+0x1560'
+    promql_expr_test:
+      - expr: 'count(ALERTS{alertstate="firing",alertname="ScheduledBackupStale"})'
+        eval_time: 26h
+        exp_samples:
+          - labels: '{}'
+            value: 2
+  - name: Capacity accounts for control-plane availability
+    interval: 1m
+    input_series:
+      - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="nonha",namespace="test"}'
+        values: '27+0x20'
+      - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="overfull",namespace="test"}'
+        values: '49+0x20'
+      - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="ha",namespace="test"}'
+        values: '27+0x20'
+      - series: 'kube_node_status_allocatable{cluster="nonha",node="large",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="nonha",node="others",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="overfull",node="large",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="overfull",node="others",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="ha",node="large",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="ha",node="others",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_role{cluster="nonha",node="a",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+      - series: 'kube_node_role{cluster="overfull",node="a",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+      - series: 'kube_node_role{cluster="ha",node="a",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+      - series: 'kube_node_role{cluster="ha",node="b",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+      - series: 'kube_node_role{cluster="ha",node="c",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+    promql_expr_test:
+      - expr: 'count(ALERTS{alertname="KubeCPUOvercommit",alertstate="firing",cluster="nonha"}) or vector(0)'
+        eval_time: 20m
+        exp_samples:
+          - labels: '{}'
+            value: 0
+      - expr: 'count(ALERTS{alertname="KubeCPUOvercommit",alertstate="firing",cluster=~"ha|overfull"})'
+        eval_time: 20m
+        exp_samples:
+          - labels: '{}'
+            value: 2

tests/run-alert-recovery.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Run alert recovery regression tests with the deployed Prometheus version.
+
+Requires PyYAML, kubectl and Podman (or a locally installed promtool).
+"""
+import pathlib
+import shutil
+import subprocess
+import tempfile
+
+import yaml
+
+root = pathlib.Path(__file__).resolve().parents[1]
+with tempfile.TemporaryDirectory(prefix="homelab-alert-tests-") as directory:
+    target = pathlib.Path(directory)
+    for name, source in (
+        ("backup", "backup-alerts.yaml"),
+        ("cpu", "cpu-capacity-alerts.yaml"),
+    ):
+        resource = yaml.safe_load(
+            (root / "apps/prometheus/resources" / source).read_text()
+        )
+        (target / f"{name}.rules.yaml").write_text(yaml.safe_dump(resource["spec"]))
+    shutil.copyfile(root / "tests/alert-recovery.test.yaml", target / "tests.yaml")
+    if shutil.which("promtool"):
+        command = ["promtool", "test", "rules", "tests.yaml"]
+    else:
+        image = subprocess.check_output(
+            ["kubectl", "-n", "prometheus", "get", "pod",
+             "prometheus-prometheus-kube-prometheus-prometheus-0",
+             "-o", "jsonpath={.spec.containers[?(@.name=='prometheus')].image}"],
+            text=True,
+        )
+        command = ["podman", "run", "--rm", "--network=none", "--user=0",
+                   "-v", f"{target}:/tests:ro", "-w", "/tests",
+                   "--entrypoint=/bin/promtool", image,
+                   "test", "rules", "tests.yaml"]
+    subprocess.run(command, cwd=target, check=True)