From e85d6aecf68e7451b812618cddeec00cd74d12e8 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Sun, 1 Nov 2026 11:21:54 +0100 Subject: [PATCH] Fix alert recovery, bound backup job history and expand data volumes --- apps/backup/README.md | 2 +- apps/backup/arschrock-postgresql.yaml | 1 + apps/backup/cephfs/base/resources.yaml | 1 + apps/backup/postgresql/base/resources.yaml | 1 + apps/backup/s3/resources.yaml | 1 + apps/elastic/resources/stack.yaml | 2 +- apps/ollama/resources/models-pvc.yaml | 16 ++++ apps/prometheus/resources/backup-alerts.yaml | 4 +- .../resources/cpu-capacity-alerts.yaml | 41 +++++++++ apps/prometheus/values.yaml | 11 +++ bootstrap/host-config/journald-retention.conf | 4 + bootstrap/ollama.yaml | 3 + docs/alert-recovery.md | 27 ++++++ tests/alert-recovery.test.yaml | 84 +++++++++++++++++++ tests/run-alert-recovery.py | 38 +++++++++ 15 files changed, 232 insertions(+), 4 deletions(-) create mode 100644 apps/ollama/resources/models-pvc.yaml create mode 100644 apps/prometheus/resources/cpu-capacity-alerts.yaml create mode 100644 bootstrap/host-config/journald-retention.conf create mode 100644 docs/alert-recovery.md create mode 100644 tests/alert-recovery.test.yaml create mode 100644 tests/run-alert-recovery.py diff --git a/apps/backup/README.md b/apps/backup/README.md index ccf3b04..0656092 100644 --- a/apps/backup/README.md +++ b/apps/backup/README.md @@ -24,6 +24,6 @@ The RGW job syncs all 12 current buckets with `rclone sync`. It reads Rook-gener ## Schedules and alerts -PostgreSQL runs daily between 02:15 and 06:30 Europe/Vienna; CephFS mirrors run between 07:00 and 12:30; RGW starts at 13:00. CronJobs forbid overlapping runs of the same backup. Prometheus alerts on failed backup Jobs, daily backup CronJobs without a success in 25 hours, and the S3 mirror without a success in 48 hours. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory. +CronJob manifests define the schedules in Europe/Vienna and forbid overlapping runs of the same backup. Prometheus alerts on failed backup Jobs, daily backup CronJobs without a success in 25 hours, and the S3 mirror without a success in 48 hours. Finished Job records and pod logs expire after one day; backup files are retained. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory. Valkey/Redis persistence is not included yet. etcd and RBD remain deferred. Existing files and old scripts directly under `/mnt/backup` are preserved. No restore has been performed; the PostgreSQL archive checks validate archive readability, not a full restore rehearsal. diff --git a/apps/backup/arschrock-postgresql.yaml b/apps/backup/arschrock-postgresql.yaml index 202a8dd..9e322dd 100644 --- a/apps/backup/arschrock-postgresql.yaml +++ b/apps/backup/arschrock-postgresql.yaml @@ -47,6 +47,7 @@ spec: jobTemplate: spec: backoffLimit: 0 + ttlSecondsAfterFinished: 86400 activeDeadlineSeconds: 14400 template: spec: diff --git a/apps/backup/cephfs/base/resources.yaml b/apps/backup/cephfs/base/resources.yaml index 6eb1e1c..04a5c1a 100644 --- a/apps/backup/cephfs/base/resources.yaml +++ b/apps/backup/cephfs/base/resources.yaml @@ -29,6 +29,7 @@ spec: jobTemplate: spec: backoffLimit: 0 + ttlSecondsAfterFinished: 86400 activeDeadlineSeconds: 21600 template: spec: diff --git a/apps/backup/postgresql/base/resources.yaml b/apps/backup/postgresql/base/resources.yaml index 6e292d6..72560cc 100644 --- a/apps/backup/postgresql/base/resources.yaml +++ b/apps/backup/postgresql/base/resources.yaml @@ -53,6 +53,7 @@ spec: jobTemplate: spec: backoffLimit: 0 + ttlSecondsAfterFinished: 86400 activeDeadlineSeconds: 21600 template: spec: diff --git a/apps/backup/s3/resources.yaml b/apps/backup/s3/resources.yaml index 3b14b7e..d1a0cbf 100644 --- a/apps/backup/s3/resources.yaml +++ b/apps/backup/s3/resources.yaml @@ -62,6 +62,7 @@ spec: jobTemplate: spec: backoffLimit: 0 + ttlSecondsAfterFinished: 86400 activeDeadlineSeconds: 172800 template: spec: diff --git a/apps/elastic/resources/stack.yaml b/apps/elastic/resources/stack.yaml index 9e3cbc1..9e02405 100644 --- a/apps/elastic/resources/stack.yaml +++ b/apps/elastic/resources/stack.yaml @@ -60,7 +60,7 @@ spec: storageClassName: rook-ceph-block-ec resources: requests: - storage: 60Gi + storage: 80Gi http: service: spec: diff --git a/apps/ollama/resources/models-pvc.yaml b/apps/ollama/resources/models-pvc.yaml new file mode 100644 index 0000000..34e9e88 --- /dev/null +++ b/apps/ollama/resources/models-pvc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ollama-models + namespace: ollama + annotations: + argocd.argoproj.io/sync-options: Prune=false,Delete=false +spec: + accessModes: + - ReadWriteOnce + storageClassName: rook-ceph-block-ec + volumeMode: Filesystem + volumeName: pvc-85f06c23-8e53-461d-94bc-b7dce0466883 + resources: + requests: + storage: 500Gi diff --git a/apps/prometheus/resources/backup-alerts.yaml b/apps/prometheus/resources/backup-alerts.yaml index 64d542e..228dec6 100644 --- a/apps/prometheus/resources/backup-alerts.yaml +++ b/apps/prometheus/resources/backup-alerts.yaml @@ -26,7 +26,7 @@ spec: or ( (kube_cronjob_info{cronjob=~".*backup.*",cronjob!="s3-backup"} - unless kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"}) + unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"}) and on(namespace,cronjob) (time() - kube_cronjob_created{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000) ) @@ -42,7 +42,7 @@ spec: or ( (kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup"} - unless kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"}) + unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"}) and on(namespace,cronjob) (time() - kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"} > 172800) ) diff --git a/apps/prometheus/resources/cpu-capacity-alerts.yaml b/apps/prometheus/resources/cpu-capacity-alerts.yaml new file mode 100644 index 0000000..8544d10 --- /dev/null +++ b/apps/prometheus/resources/cpu-capacity-alerts.yaml @@ -0,0 +1,41 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: cpu-capacity + namespace: prometheus + labels: + release: prometheus +spec: + groups: + - name: cpu-capacity + rules: + - alert: KubeCPUOvercommit + # The upstream HA branch also fires on single-control-plane clusters. + # Only require largest-node failover capacity when the control plane is HA. + expr: |- + ( + ( + sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum) + - sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) + ) > 0 + and on (cluster) + count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) < 3 + ) + or + ( + ( + sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum) + - ( + sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) + - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) + ) + ) > 0 + and on (cluster) + count by (cluster) (max by (cluster,node) (kube_node_role{job="kube-state-metrics",role="control-plane"})) >= 3 + ) + for: 10m + labels: + severity: warning + annotations: + summary: Cluster has overcommitted CPU resource requests. + description: "CPU requests exceed usable capacity by {{ $value }} cores (total capacity for a non-HA control plane; largest-node failure capacity for HA)." diff --git a/apps/prometheus/values.yaml b/apps/prometheus/values.yaml index df9e0aa..2ab8ba2 100644 --- a/apps/prometheus/values.yaml +++ b/apps/prometheus/values.yaml @@ -2,9 +2,20 @@ grafana: enabled: false defaultRules: + disabled: + KubeCPUOvercommit: true rules: kubeProxy: false +prometheus-node-exporter: + prometheus: + monitor: + metricRelabelings: + # These bind mounts share /mnt's capacity; retain its canonical gauges. + - sourceLabels: [__name__, fstype, mountpoint] + regex: 'node_filesystem_.+;bcachefs;(/komposthaufen|/var/www|/var/lib/(lidarr|prowlarr|radarr|readarr|sonarr|whisparr)|/home/feedc0de/(\.steam/steamapps|mcserver/plugins/dynmap/web/tiles))' + action: drop + alertmanager: ingress: enabled: true diff --git a/bootstrap/host-config/journald-retention.conf b/bootstrap/host-config/journald-retention.conf new file mode 100644 index 0000000..13e40dc --- /dev/null +++ b/bootstrap/host-config/journald-retention.conf @@ -0,0 +1,4 @@ +[Journal] +SystemMaxUse=256M +SystemKeepFree=2G +MaxRetentionSec=7day diff --git a/bootstrap/ollama.yaml b/bootstrap/ollama.yaml index 827fcc7..6fc5e9f 100644 --- a/bootstrap/ollama.yaml +++ b/bootstrap/ollama.yaml @@ -18,6 +18,9 @@ spec: - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git targetRevision: main ref: values + - repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git + targetRevision: main + path: apps/ollama/resources destination: server: https://kubernetes.default.svc namespace: ollama diff --git a/docs/alert-recovery.md b/docs/alert-recovery.md new file mode 100644 index 0000000..6b853cb --- /dev/null +++ b/docs/alert-recovery.md @@ -0,0 +1,27 @@ +# Alert recovery + +Backup success is the CronJob's `lastSuccessfulTime`, not the continued +existence of an older failed Job. Backup rules must match CronJob info and +success metrics explicitly on `(namespace,cronjob)`; info has extra schedule +labels. Test both recovery and real failures with +`python3 tests/run-alert-recovery.py` (PyYAML and promtool or Podman required). + +Backup Job metadata and pod logs expire one day after completion or failure. +This does not delete backup files. Stale-backup rules continue to alert when +successful backups are missing, even after failed Job records expire. +During recovery, retire old failed Jobs only after confirming a newer success +for their owning CronJob. Do not mark a backup successful manually. + +The CPU-capacity alert preserves the upstream total-capacity check for a +non-HA control plane and applies largest-node failure capacity only to HA +control planes. Requests remain based on the review in `cpu-sizing.md`. +This does not promise failover for workloads pinned to arschrock. + +On odroid-h3-04, persistent journald storage is bounded using +`bootstrap/host-config/journald-retention.conf`, installed as +`/etc/systemd/journald.conf.d/homelab-retention.conf`. This host configuration +requires installation through SSH; Argo does not manage host systemd files. +The node's small system disk also holds active container images, so deleting +unused images alone may free nothing. Never remove Ceph monitor database files +to make space. Ceph crash reports may be archived after reviewing the cause +and verifying it has stopped; archival retains the reports for investigation. diff --git a/tests/alert-recovery.test.yaml b/tests/alert-recovery.test.yaml new file mode 100644 index 0000000..8d79033 --- /dev/null +++ b/tests/alert-recovery.test.yaml @@ -0,0 +1,84 @@ +# Rule files are generated from the PrometheusRule manifests by the test runner. +rule_files: + - backup.rules.yaml + - cpu.rules.yaml +evaluation_interval: 1m +tests: + - name: Successful backups match CronJob info despite its extra schedule labels + interval: 1m + input_series: + - series: 'kube_cronjob_info{namespace="test",cronjob="postgresql-backup",schedule="0 2 * * *",concurrency_policy="Forbid"}' + values: '1+0x4320' + - series: 'kube_cronjob_created{namespace="test",cronjob="postgresql-backup"}' + values: '0+0x4320' + - series: 'kube_cronjob_status_last_successful_time{namespace="test",cronjob="postgresql-backup"}' + values: '255600+0x4320' + - series: 'kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup",schedule="0 16 * * *",concurrency_policy="Forbid"}' + values: '1+0x4320' + - series: 'kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"}' + values: '0+0x4320' + - series: 'kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"}' + values: '255600+0x4320' + promql_expr_test: + - expr: 'count(ALERTS{alertstate="firing",alertname=~"ScheduledBackupStale|S3BackupStale"}) or vector(0)' + eval_time: 72h + exp_samples: + - labels: '{}' + value: 0 + - name: Stale and never successful backups still alert + interval: 1m + input_series: + - series: 'kube_cronjob_info{namespace="test",cronjob="never-backup",schedule="0 2 * * *"}' + values: '1+0x1560' + - series: 'kube_cronjob_created{namespace="test",cronjob="never-backup"}' + values: '0+0x1560' + - series: 'kube_cronjob_status_last_successful_time{namespace="test",cronjob="stale-backup"}' + values: '0+0x1560' + promql_expr_test: + - expr: 'count(ALERTS{alertstate="firing",alertname="ScheduledBackupStale"})' + eval_time: 26h + exp_samples: + - labels: '{}' + value: 2 + - name: Capacity accounts for control-plane availability + interval: 1m + input_series: + - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="nonha",namespace="test"}' + values: '27+0x20' + - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="overfull",namespace="test"}' + values: '49+0x20' + - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="ha",namespace="test"}' + values: '27+0x20' + - series: 'kube_node_status_allocatable{cluster="nonha",node="large",job="kube-state-metrics",resource="cpu"}' + values: '24+0x20' + - series: 'kube_node_status_allocatable{cluster="nonha",node="others",job="kube-state-metrics",resource="cpu"}' + values: '24+0x20' + - series: 'kube_node_status_allocatable{cluster="overfull",node="large",job="kube-state-metrics",resource="cpu"}' + values: '24+0x20' + - series: 'kube_node_status_allocatable{cluster="overfull",node="others",job="kube-state-metrics",resource="cpu"}' + values: '24+0x20' + - series: 'kube_node_status_allocatable{cluster="ha",node="large",job="kube-state-metrics",resource="cpu"}' + values: '24+0x20' + - series: 'kube_node_status_allocatable{cluster="ha",node="others",job="kube-state-metrics",resource="cpu"}' + values: '24+0x20' + - series: 'kube_node_role{cluster="nonha",node="a",job="kube-state-metrics",role="control-plane"}' + values: '1+0x20' + - series: 'kube_node_role{cluster="overfull",node="a",job="kube-state-metrics",role="control-plane"}' + values: '1+0x20' + - series: 'kube_node_role{cluster="ha",node="a",job="kube-state-metrics",role="control-plane"}' + values: '1+0x20' + - series: 'kube_node_role{cluster="ha",node="b",job="kube-state-metrics",role="control-plane"}' + values: '1+0x20' + - series: 'kube_node_role{cluster="ha",node="c",job="kube-state-metrics",role="control-plane"}' + values: '1+0x20' + promql_expr_test: + - expr: 'count(ALERTS{alertname="KubeCPUOvercommit",alertstate="firing",cluster="nonha"}) or vector(0)' + eval_time: 20m + exp_samples: + - labels: '{}' + value: 0 + - expr: 'count(ALERTS{alertname="KubeCPUOvercommit",alertstate="firing",cluster=~"ha|overfull"})' + eval_time: 20m + exp_samples: + - labels: '{}' + value: 2 diff --git a/tests/run-alert-recovery.py b/tests/run-alert-recovery.py new file mode 100644 index 0000000..24e9263 --- /dev/null +++ b/tests/run-alert-recovery.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Run alert recovery regression tests with the deployed Prometheus version. + +Requires PyYAML, kubectl and Podman (or a locally installed promtool). +""" +import pathlib +import shutil +import subprocess +import tempfile + +import yaml + +root = pathlib.Path(__file__).resolve().parents[1] +with tempfile.TemporaryDirectory(prefix="homelab-alert-tests-") as directory: + target = pathlib.Path(directory) + for name, source in ( + ("backup", "backup-alerts.yaml"), + ("cpu", "cpu-capacity-alerts.yaml"), + ): + resource = yaml.safe_load( + (root / "apps/prometheus/resources" / source).read_text() + ) + (target / f"{name}.rules.yaml").write_text(yaml.safe_dump(resource["spec"])) + shutil.copyfile(root / "tests/alert-recovery.test.yaml", target / "tests.yaml") + if shutil.which("promtool"): + command = ["promtool", "test", "rules", "tests.yaml"] + else: + image = subprocess.check_output( + ["kubectl", "-n", "prometheus", "get", "pod", + "prometheus-prometheus-kube-prometheus-prometheus-0", + "-o", "jsonpath={.spec.containers[?(@.name=='prometheus')].image}"], + text=True, + ) + command = ["podman", "run", "--rm", "--network=none", "--user=0", + "-v", f"{target}:/tests:ro", "-w", "/tests", + "--entrypoint=/bin/promtool", image, + "test", "rules", "tests.yaml"] + subprocess.run(command, cwd=target, check=True)