Fix alert recovery, bound backup job history and expand data volumes

apps/backup/README.md

@@ -24,6 +24,6 @@ The RGW job syncs all 12 current buckets with `rclone sync`. It reads Rook-gener
 
 ## Schedules and alerts
 
-PostgreSQL runs daily between 02:15 and 06:30 Europe/Vienna; CephFS mirrors run between 07:00 and 12:30; RGW starts at 13:00. CronJobs forbid overlapping runs of the same backup. Prometheus alerts on failed backup Jobs, daily backup CronJobs without a success in 25 hours, and the S3 mirror without a success in 48 hours. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory.
+CronJob manifests define the schedules in Europe/Vienna and forbid overlapping runs of the same backup. Prometheus alerts on failed backup Jobs, daily backup CronJobs without a success in 25 hours, and the S3 mirror without a success in 48 hours. Finished Job records and pod logs expire after one day; backup files are retained. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory.
 
 Valkey/Redis persistence is not included yet. etcd and RBD remain deferred. Existing files and old scripts directly under `/mnt/backup` are preserved. No restore has been performed; the PostgreSQL archive checks validate archive readability, not a full restore rehearsal.

apps/backup/arschrock-postgresql.yaml

@@ -47,6 +47,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 0
+      ttlSecondsAfterFinished: 86400
       activeDeadlineSeconds: 14400
       template:
         spec:

apps/backup/cephfs/base/resources.yaml

@@ -29,6 +29,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 0
+      ttlSecondsAfterFinished: 86400
       activeDeadlineSeconds: 21600
       template:
         spec:

apps/backup/postgresql/base/resources.yaml

@@ -53,6 +53,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 0
+      ttlSecondsAfterFinished: 86400
       activeDeadlineSeconds: 21600
       template:
         spec:

apps/backup/s3/resources.yaml

@@ -62,6 +62,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 0
+      ttlSecondsAfterFinished: 86400
       activeDeadlineSeconds: 172800
       template:
         spec:

apps/elastic/resources/stack.yaml

@@ -60,7 +60,7 @@ spec:
             storageClassName: rook-ceph-block-ec
             resources:
               requests:
-                storage: 60Gi
+                storage: 80Gi
   http:
     service:
       spec:

apps/ollama/resources/models-pvc.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ollama-models
+  namespace: ollama
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false,Delete=false
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: rook-ceph-block-ec
+  volumeMode: Filesystem
+  volumeName: pvc-85f06c23-8e53-461d-94bc-b7dce0466883
+  resources:
+    requests:
+      storage: 500Gi

apps/prometheus/resources/backup-alerts.yaml

@@ -26,7 +26,7 @@ spec:
             or
             (
               (kube_cronjob_info{cronjob=~".*backup.*",cronjob!="s3-backup"}
-                unless kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
+                unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
               and on(namespace,cronjob)
               (time() - kube_cronjob_created{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000)
             )
@@ -42,7 +42,7 @@ spec:
             or
             (
               (kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup"}
-                unless kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
+                unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
               and on(namespace,cronjob)
               (time() - kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"} > 172800)
             )

apps/prometheus/resources/cpu-capacity-alerts.yaml

@@ -0,0 +1,41 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cpu-capacity
+  namespace: prometheus
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: cpu-capacity
+      rules:
+        - alert: KubeCPUOvercommit
+          # The upstream HA branch also fires on single-control-plane clusters.
+          # Only require largest-node failover capacity when the control plane is HA.
+          expr: |-
+            (
+              (
+                sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum)
+                - sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+              ) > 0
+              and on (cluster)
+              count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) < 3
+            )
+            or
+            (
+              (
+                sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum)
+                - (
+                  sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+                  - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+                )
+              ) > 0
+              and on (cluster)
+              count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) >= 3
+            )
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: Cluster has overcommitted CPU resource requests.
+            description: "CPU requests exceed usable capacity by {{ $value }} cores (total capacity for a non-HA control plane; largest-node failure capacity for HA)."

apps/prometheus/values.yaml

@@ -2,9 +2,20 @@ grafana:
   enabled: false
 
 defaultRules:
+  disabled:
+    KubeCPUOvercommit: true
   rules:
     kubeProxy: false
 
+prometheus-node-exporter:
+  prometheus:
+    monitor:
+      metricRelabelings:
+        # These bind mounts share /mnt's capacity; retain its canonical gauges.
+        - sourceLabels: [__name__, fstype, mountpoint]
+          regex: 'node_filesystem_.+;bcachefs;(/komposthaufen|/var/www|/var/lib/(lidarr|prowlarr|radarr|readarr|sonarr|whisparr)|/home/feedc0de/(\.steam/steamapps|mcserver/plugins/dynmap/web/tiles))'
+          action: drop
+
 alertmanager:
   ingress:
     enabled: true