Fix alert recovery, bound backup job history and expand data volumes

apps/prometheus/resources/backup-alerts.yaml

@@ -26,7 +26,7 @@ spec:
             or
             (
               (kube_cronjob_info{cronjob=~".*backup.*",cronjob!="s3-backup"}
-                unless kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
+                unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
               and on(namespace,cronjob)
               (time() - kube_cronjob_created{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000)
             )
@@ -42,7 +42,7 @@ spec:
             or
             (
               (kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup"}
-                unless kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
+                unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
               and on(namespace,cronjob)
               (time() - kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"} > 172800)
             )

apps/prometheus/resources/cpu-capacity-alerts.yaml

@@ -0,0 +1,41 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cpu-capacity
+  namespace: prometheus
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: cpu-capacity
+      rules:
+        - alert: KubeCPUOvercommit
+          # The upstream HA branch also fires on single-control-plane clusters.
+          # Only require largest-node failover capacity when the control plane is HA.
+          expr: |-
+            (
+              (
+                sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum)
+                - sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+              ) > 0
+              and on (cluster)
+              count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) < 3
+            )
+            or
+            (
+              (
+                sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum)
+                - (
+                  sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+                  - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
+                )
+              ) > 0
+              and on (cluster)
+              count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) >= 3
+            )
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: Cluster has overcommitted CPU resource requests.
+            description: "CPU requests exceed usable capacity by {{ $value }} cores (total capacity for a non-HA control plane; largest-node failure capacity for HA)."