Fix alert recovery, bound backup job history and expand data volumes

apps/prometheus/resources/backup-alerts.yaml

@@ -26,7 +26,7 @@ spec:
             or
             (
               (kube_cronjob_info{cronjob=~".*backup.*",cronjob!="s3-backup"}
-                unless kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
+                unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
               and on(namespace,cronjob)
               (time() - kube_cronjob_created{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000)
             )
@@ -42,7 +42,7 @@ spec:
             or
             (
               (kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup"}
-                unless kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
+                unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
               and on(namespace,cronjob)
               (time() - kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"} > 172800)
             )

apps/prometheus/resources/cpu-capacity-alerts.yaml

@@ -0,0 +1,41 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cpu-capacity
+  namespace: prometheus
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: cpu-capacity
+      rules:
+        - alert: KubeCPUOvercommit
+          # The upstream HA branch also fires on single-control-plane clusters.
+          # Only require largest-node failover capacity when the control plane is HA.
+          expr: |-
+            (
+              (
+                sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum)
+                - sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+              ) > 0
+              and on (cluster)
+              count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) < 3
+            )
+            or
+            (
+              (
+                sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum)
+                - (
+                  sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+                  - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+                )
+              ) > 0
+              and on (cluster)
+              count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) >= 3
+            )
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: Cluster has overcommitted CPU resource requests.
+            description: "CPU requests exceed usable capacity by {{ $value }} cores (total capacity for a non-HA control plane; largest-node failure capacity for HA)."

apps/prometheus/values.yaml

@@ -2,9 +2,20 @@ grafana:
   enabled: false
 
 defaultRules:
+  disabled:
+    KubeCPUOvercommit: true
   rules:
     kubeProxy: false
 
+prometheus-node-exporter:
+  prometheus:
+    monitor:
+      metricRelabelings:
+        # These bind mounts share /mnt's capacity; retain its canonical gauges.
+        - sourceLabels: [__name__, fstype, mountpoint]
+          regex: 'node_filesystem_.+;bcachefs;(/komposthaufen|/var/www|/var/lib/(lidarr|prowlarr|radarr|readarr|sonarr|whisparr)|/home/feedc0de/(\.steam/steamapps|mcserver/plugins/dynmap/web/tiles))'
+          action: drop
+
 alertmanager:
   ingress:
     enabled: true