Fix alert recovery, bound backup job history and expand data volumes

tests/alert-recovery.test.yaml

@@ -0,0 +1,84 @@
+# Rule files are generated from the PrometheusRule manifests by the test runner.
+rule_files:
+  - backup.rules.yaml
+  - cpu.rules.yaml
+evaluation_interval: 1m
+tests:
+  - name: Successful backups match CronJob info despite its extra schedule labels
+    interval: 1m
+    input_series:
+      - series: 'kube_cronjob_info{namespace="test",cronjob="postgresql-backup",schedule="0 2 * * *",concurrency_policy="Forbid"}'
+        values: '1+0x4320'
+      - series: 'kube_cronjob_created{namespace="test",cronjob="postgresql-backup"}'
+        values: '0+0x4320'
+      - series: 'kube_cronjob_status_last_successful_time{namespace="test",cronjob="postgresql-backup"}'
+        values: '255600+0x4320'
+      - series: 'kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup",schedule="0 16 * * *",concurrency_policy="Forbid"}'
+        values: '1+0x4320'
+      - series: 'kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"}'
+        values: '0+0x4320'
+      - series: 'kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"}'
+        values: '255600+0x4320'
+    promql_expr_test:
+      - expr: 'count(ALERTS{alertstate="firing",alertname=~"ScheduledBackupStale|S3BackupStale"}) or vector(0)'
+        eval_time: 72h
+        exp_samples:
+          - labels: '{}'
+            value: 0
+  - name: Stale and never successful backups still alert
+    interval: 1m
+    input_series:
+      - series: 'kube_cronjob_info{namespace="test",cronjob="never-backup",schedule="0 2 * * *"}'
+        values: '1+0x1560'
+      - series: 'kube_cronjob_created{namespace="test",cronjob="never-backup"}'
+        values: '0+0x1560'
+      - series: 'kube_cronjob_status_last_successful_time{namespace="test",cronjob="stale-backup"}'
+        values: '0+0x1560'
+    promql_expr_test:
+      - expr: 'count(ALERTS{alertstate="firing",alertname="ScheduledBackupStale"})'
+        eval_time: 26h
+        exp_samples:
+          - labels: '{}'
+            value: 2
+  - name: Capacity accounts for control-plane availability
+    interval: 1m
+    input_series:
+      - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="nonha",namespace="test"}'
+        values: '27+0x20'
+      - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="overfull",namespace="test"}'
+        values: '49+0x20'
+      - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="ha",namespace="test"}'
+        values: '27+0x20'
+      - series: 'kube_node_status_allocatable{cluster="nonha",node="large",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="nonha",node="others",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="overfull",node="large",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="overfull",node="others",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="ha",node="large",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_status_allocatable{cluster="ha",node="others",job="kube-state-metrics",resource="cpu"}'
+        values: '24+0x20'
+      - series: 'kube_node_role{cluster="nonha",node="a",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+      - series: 'kube_node_role{cluster="overfull",node="a",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+      - series: 'kube_node_role{cluster="ha",node="a",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+      - series: 'kube_node_role{cluster="ha",node="b",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+      - series: 'kube_node_role{cluster="ha",node="c",job="kube-state-metrics",role="control-plane"}'
+        values: '1+0x20'
+    promql_expr_test:
+      - expr: 'count(ALERTS{alertname="KubeCPUOvercommit",alertstate="firing",cluster="nonha"}) or vector(0)'
+        eval_time: 20m
+        exp_samples:
+          - labels: '{}'
+            value: 0
+      - expr: 'count(ALERTS{alertname="KubeCPUOvercommit",alertstate="firing",cluster=~"ha|overfull"})'
+        eval_time: 20m
+        exp_samples:
+          - labels: '{}'
+            value: 2

tests/run-alert-recovery.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Run alert recovery regression tests with the deployed Prometheus version.
+
+Requires PyYAML, kubectl and Podman (or a locally installed promtool).
+"""
+import pathlib
+import shutil
+import subprocess
+import tempfile
+
+import yaml
+
+root = pathlib.Path(__file__).resolve().parents[1]
+with tempfile.TemporaryDirectory(prefix="homelab-alert-tests-") as directory:
+    target = pathlib.Path(directory)
+    for name, source in (
+        ("backup", "backup-alerts.yaml"),
+        ("cpu", "cpu-capacity-alerts.yaml"),
+    ):
+        resource = yaml.safe_load(
+            (root / "apps/prometheus/resources" / source).read_text()
+        )
+        (target / f"{name}.rules.yaml").write_text(yaml.safe_dump(resource["spec"]))
+    shutil.copyfile(root / "tests/alert-recovery.test.yaml", target / "tests.yaml")
+    if shutil.which("promtool"):
+        command = ["promtool", "test", "rules", "tests.yaml"]
+    else:
+        image = subprocess.check_output(
+            ["kubectl", "-n", "prometheus", "get", "pod",
+             "prometheus-prometheus-kube-prometheus-prometheus-0",
+             "-o", "jsonpath={.spec.containers[?(@.name=='prometheus')].image}"],
+            text=True,
+        )
+        command = ["podman", "run", "--rm", "--network=none", "--user=0",
+                   "-v", f"{target}:/tests:ro", "-w", "/tests",
+                   "--entrypoint=/bin/promtool", image,
+                   "test", "rules", "tests.yaml"]
+    subprocess.run(command, cwd=target, check=True)