From 53e9646061226044a47de7a0e30dc4be59e01fbf Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Sat, 24 Oct 2026 20:16:14 +0200 Subject: [PATCH] Add guarded arschrock PostgreSQL backup CronJob and alerts --- apps/backup/README.md | 7 ++ apps/backup/arschrock-postgresql.yaml | 88 ++++++++++++++++++++ apps/backup/kustomization.yaml | 4 + apps/prometheus/resources/backup-alerts.yaml | 27 ++++++ bootstrap/backup-namespace.yaml | 4 + bootstrap/backup-project.yaml | 17 ++++ bootstrap/backup.yaml | 19 +++++ bootstrap/kustomization.yaml | 3 + 8 files changed, 169 insertions(+) create mode 100644 apps/backup/README.md create mode 100644 apps/backup/arschrock-postgresql.yaml create mode 100644 apps/backup/kustomization.yaml create mode 100644 apps/prometheus/resources/backup-alerts.yaml create mode 100644 bootstrap/backup-namespace.yaml create mode 100644 bootstrap/backup-project.yaml create mode 100644 bootstrap/backup.yaml diff --git a/apps/backup/README.md b/apps/backup/README.md new file mode 100644 index 0000000..689f794 --- /dev/null +++ b/apps/backup/README.md @@ -0,0 +1,7 @@ +# Cluster backups + +The first backup schedule dumps every database and the global roles from the host PostgreSQL server on arschrock. It uses the host Unix socket and peer authentication; no database password or fixed ClusterIP is stored in Git. The pod is pinned to arschrock and checks that `/mnt/backup` is on bcachefs before creating output. Kubernetes uses `hostPath.type: Directory`, never `DirectoryOrCreate`. + +One-time host preparation: `sudo install -d -o 962 -g 962 -m 0700 /mnt/backup/kubernetes/postgresql/arschrock`. Verify UID 962 still belongs to the host `postgres` user before changing it. Existing files directly under `/mnt/backup` are untouched. Each run writes a timestamped directory with `globals.sql`, one custom-format dump per database, and a `complete` marker written only after all dumps validate with `pg_restore --list`. A directory without `complete` is incomplete and must not be used for restore. No automatic retention or deletion is enabled yet. + +The Prometheus rules in `apps/prometheus/resources/backup-alerts.yaml` alert on failed jobs and a missing or stale successful host PostgreSQL backup. This is **not yet a complete cluster backup**: in-cluster PostgreSQL, Valkey, CephFS, and RGW/S3 still need schedules, credential handling, and restore tests. etcd and RBD were explicitly deferred. The old scripts and files in `/mnt/backup` are intentionally preserved. diff --git a/apps/backup/arschrock-postgresql.yaml b/apps/backup/arschrock-postgresql.yaml new file mode 100644 index 0000000..a53a2c6 --- /dev/null +++ b/apps/backup/arschrock-postgresql.yaml @@ -0,0 +1,88 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: arschrock-postgresql-backup + namespace: backup +data: + backup.sh: | + #!/bin/sh + set -eu + test "$(stat -f -c %T /backup)" = bcachefs + test -d /backup/kubernetes/postgresql/arschrock + test "$(stat -c %u /backup/kubernetes/postgresql/arschrock)" = 962 + test -S /postgresql/.s.PGSQL.5432 + export PGHOST=/postgresql PGUSER=postgres PGDATABASE=postgres + destination="/backup/kubernetes/postgresql/arschrock/$(date -u +%Y-%m-%dT%H-%M-%SZ)" + mkdir -m 0700 "$destination" + pg_dumpall --globals-only > "$destination/globals.sql.partial" + test -s "$destination/globals.sql.partial" + mv "$destination/globals.sql.partial" "$destination/globals.sql" + psql -AtX -v ON_ERROR_STOP=1 -c 'SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate ORDER BY datname' > "$destination/databases.list" + while IFS= read -r database; do + case "$database" in + *[!a-zA-Z0-9_-]*|'') echo "Unsafe database name: $database" >&2; exit 1 ;; + esac + pg_dump --format=custom --file="$destination/$database.dump.partial" --dbname="$database" + pg_restore --list "$destination/$database.dump.partial" >/dev/null + mv "$destination/$database.dump.partial" "$destination/$database.dump" + done < "$destination/databases.list" + test "$(find "$destination" -maxdepth 1 -name '*.dump' | wc -l)" -gt 0 + date -u +%FT%TZ > "$destination/complete" + echo "Completed PostgreSQL backup at $destination" +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: arschrock-postgresql-backup + namespace: backup +spec: + schedule: "15 2 * * *" + timeZone: Europe/Vienna + concurrencyPolicy: Forbid + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 7 + failedJobsHistoryLimit: 7 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 14400 + template: + spec: + nodeSelector: + kubernetes.io/hostname: arschrock + restartPolicy: Never + securityContext: + runAsUser: 962 + runAsGroup: 962 + containers: + - name: pg-dump + image: postgres:18.6 + imagePullPolicy: IfNotPresent + command: [/bin/sh, /scripts/backup.sh] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: backup + mountPath: /backup + - name: postgresql-socket + mountPath: /postgresql + readOnly: true + - name: scripts + mountPath: /scripts + readOnly: true + volumes: + - name: backup + hostPath: + path: /mnt/backup + type: Directory + - name: postgresql-socket + hostPath: + path: /run/postgresql + type: Directory + - name: scripts + configMap: + name: arschrock-postgresql-backup + defaultMode: 0555 diff --git a/apps/backup/kustomization.yaml b/apps/backup/kustomization.yaml new file mode 100644 index 0000000..aa36412 --- /dev/null +++ b/apps/backup/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - arschrock-postgresql.yaml diff --git a/apps/prometheus/resources/backup-alerts.yaml b/apps/prometheus/resources/backup-alerts.yaml new file mode 100644 index 0000000..3403998 --- /dev/null +++ b/apps/prometheus/resources/backup-alerts.yaml @@ -0,0 +1,27 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: backup-jobs + namespace: prometheus + labels: + release: prometheus +spec: + groups: + - name: backup-jobs + rules: + - alert: BackupCronJobFailed + expr: kube_job_status_failed{namespace="backup"} > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Backup job failed" + description: "{{ $labels.namespace }}/{{ $labels.job_name }} failed; inspect its pod logs and /mnt/backup mount." + - alert: ArschrockPostgreSQLBackupStale + expr: (time() - kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"} > 90000) or (kube_cronjob_info{namespace="backup",cronjob="arschrock-postgresql-backup"} unless kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"}) + for: 10m + labels: + severity: critical + annotations: + summary: "Arschrock PostgreSQL backup has no recent successful run" + description: "The daily PostgreSQL dump is older than 25 hours or has never succeeded." diff --git a/bootstrap/backup-namespace.yaml b/bootstrap/backup-namespace.yaml new file mode 100644 index 0000000..ce4bab1 --- /dev/null +++ b/bootstrap/backup-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: backup diff --git a/bootstrap/backup-project.yaml b/bootstrap/backup-project.yaml new file mode 100644 index 0000000..750dad3 --- /dev/null +++ b/bootstrap/backup-project.yaml @@ -0,0 +1,17 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: backup + namespace: argocd +spec: + description: Cluster-specific backup schedules and storage access + sourceRepos: + - https://code.brunner.ninja/feedc0de/homelab-gitops.git + destinations: + - namespace: backup + server: https://kubernetes.default.svc + namespaceResourceWhitelist: + - group: "" + kind: ConfigMap + - group: batch + kind: CronJob diff --git a/bootstrap/backup.yaml b/bootstrap/backup.yaml new file mode 100644 index 0000000..d470265 --- /dev/null +++ b/bootstrap/backup.yaml @@ -0,0 +1,19 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: backup + namespace: argocd +spec: + project: backup + source: + repoURL: https://code.brunner.ninja/feedc0de/homelab-gitops.git + targetRevision: main + path: apps/backup + destination: + server: https://kubernetes.default.svc + namespace: backup + syncPolicy: + automated: + enabled: true + prune: false + selfHeal: true diff --git a/bootstrap/kustomization.yaml b/bootstrap/kustomization.yaml index 04b3f71..826c255 100644 --- a/bootstrap/kustomization.yaml +++ b/bootstrap/kustomization.yaml @@ -1,6 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: +- backup-namespace.yaml +- backup-project.yaml +- backup.yaml - argocd-project.yaml - argocd.yaml - project.yaml