Add guarded arschrock PostgreSQL backup CronJob and alerts

apps/backup/README.md

@@ -0,0 +1,7 @@
+# Cluster backups
+
+The first backup schedule dumps every database and the global roles from the host PostgreSQL server on arschrock. It uses the host Unix socket and peer authentication; no database password or fixed ClusterIP is stored in Git. The pod is pinned to arschrock and checks that `/mnt/backup` is on bcachefs before creating output. Kubernetes uses `hostPath.type: Directory`, never `DirectoryOrCreate`.
+
+One-time host preparation: `sudo install -d -o 962 -g 962 -m 0700 /mnt/backup/kubernetes/postgresql/arschrock`. Verify UID 962 still belongs to the host `postgres` user before changing it. Existing files directly under `/mnt/backup` are untouched. Each run writes a timestamped directory with `globals.sql`, one custom-format dump per database, and a `complete` marker written only after all dumps validate with `pg_restore --list`. A directory without `complete` is incomplete and must not be used for restore. No automatic retention or deletion is enabled yet.
+
+The Prometheus rules in `apps/prometheus/resources/backup-alerts.yaml` alert on failed jobs and a missing or stale successful host PostgreSQL backup. This is **not yet a complete cluster backup**: in-cluster PostgreSQL, Valkey, CephFS, and RGW/S3 still need schedules, credential handling, and restore tests. etcd and RBD were explicitly deferred. The old scripts and files in `/mnt/backup` are intentionally preserved.

apps/backup/arschrock-postgresql.yaml

@@ -0,0 +1,88 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: arschrock-postgresql-backup
+  namespace: backup
+data:
+  backup.sh: |
+    #!/bin/sh
+    set -eu
+    test "$(stat -f -c %T /backup)" = bcachefs
+    test -d /backup/kubernetes/postgresql/arschrock
+    test "$(stat -c %u /backup/kubernetes/postgresql/arschrock)" = 962
+    test -S /postgresql/.s.PGSQL.5432
+    export PGHOST=/postgresql PGUSER=postgres PGDATABASE=postgres
+    destination="/backup/kubernetes/postgresql/arschrock/$(date -u +%Y-%m-%dT%H-%M-%SZ)"
+    mkdir -m 0700 "$destination"
+    pg_dumpall --globals-only > "$destination/globals.sql.partial"
+    test -s "$destination/globals.sql.partial"
+    mv "$destination/globals.sql.partial" "$destination/globals.sql"
+    psql -AtX -v ON_ERROR_STOP=1 -c 'SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate ORDER BY datname' > "$destination/databases.list"
+    while IFS= read -r database; do
+        case "$database" in
+          *[!a-zA-Z0-9_-]*|'') echo "Unsafe database name: $database" >&2; exit 1 ;;
+        esac
+        pg_dump --format=custom --file="$destination/$database.dump.partial" --dbname="$database"
+        pg_restore --list "$destination/$database.dump.partial" >/dev/null
+        mv "$destination/$database.dump.partial" "$destination/$database.dump"
+    done < "$destination/databases.list"
+    test "$(find "$destination" -maxdepth 1 -name '*.dump' | wc -l)" -gt 0
+    date -u +%FT%TZ > "$destination/complete"
+    echo "Completed PostgreSQL backup at $destination"
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: arschrock-postgresql-backup
+  namespace: backup
+spec:
+  schedule: "15 2 * * *"
+  timeZone: Europe/Vienna
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 3600
+  successfulJobsHistoryLimit: 7
+  failedJobsHistoryLimit: 7
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 14400
+      template:
+        spec:
+          nodeSelector:
+            kubernetes.io/hostname: arschrock
+          restartPolicy: Never
+          securityContext:
+            runAsUser: 962
+            runAsGroup: 962
+          containers:
+            - name: pg-dump
+              image: postgres:18.6
+              imagePullPolicy: IfNotPresent
+              command: [/bin/sh, /scripts/backup.sh]
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: [ALL]
+              volumeMounts:
+                - name: backup
+                  mountPath: /backup
+                - name: postgresql-socket
+                  mountPath: /postgresql
+                  readOnly: true
+                - name: scripts
+                  mountPath: /scripts
+                  readOnly: true
+          volumes:
+            - name: backup
+              hostPath:
+                path: /mnt/backup
+                type: Directory
+            - name: postgresql-socket
+              hostPath:
+                path: /run/postgresql
+                type: Directory
+            - name: scripts
+              configMap:
+                name: arschrock-postgresql-backup
+                defaultMode: 0555

apps/backup/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - arschrock-postgresql.yaml

apps/prometheus/resources/backup-alerts.yaml

@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: backup-jobs
+  namespace: prometheus
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: backup-jobs
+      rules:
+        - alert: BackupCronJobFailed
+          expr: kube_job_status_failed{namespace="backup"} > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Backup job failed"
+            description: "{{ $labels.namespace }}/{{ $labels.job_name }} failed; inspect its pod logs and /mnt/backup mount."
+        - alert: ArschrockPostgreSQLBackupStale
+          expr: (time() - kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"} > 90000) or (kube_cronjob_info{namespace="backup",cronjob="arschrock-postgresql-backup"} unless kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"})
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Arschrock PostgreSQL backup has no recent successful run"
+            description: "The daily PostgreSQL dump is older than 25 hours or has never succeeded."

bootstrap/backup-namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: backup

bootstrap/backup-project.yaml

@@ -0,0 +1,17 @@
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: backup
+  namespace: argocd
+spec:
+  description: Cluster-specific backup schedules and storage access
+  sourceRepos:
+    - https://code.brunner.ninja/feedc0de/homelab-gitops.git
+  destinations:
+    - namespace: backup
+      server: https://kubernetes.default.svc
+  namespaceResourceWhitelist:
+    - group: ""
+      kind: ConfigMap
+    - group: batch
+      kind: CronJob

bootstrap/backup.yaml

@@ -0,0 +1,19 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: backup
+  namespace: argocd
+spec:
+  project: backup
+  source:
+    repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git
+    targetRevision: main
+    path: apps/backup
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: backup
+  syncPolicy:
+    automated:
+      enabled: true
+      prune: false
+      selfHeal: true

bootstrap/kustomization.yaml

@@ -1,6 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+- backup-namespace.yaml
+- backup-project.yaml
+- backup.yaml
 - argocd-project.yaml
 - argocd.yaml
 - project.yaml