From ca02186ad1000d9f77419f0a01250a007879c6c3 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Sun, 25 Oct 2026 07:08:16 +0100 Subject: [PATCH] Add PostgreSQL, CephFS and RGW backup jobs --- apps/backup/README.md | 20 +- apps/backup/arschrock-postgresql.yaml | 2 +- apps/backup/cephfs/base/kustomization.yaml | 4 + apps/backup/cephfs/base/resources.yaml | 78 ++++++++ .../default-brgkepler-com/kustomization.yaml | 25 +++ .../default-brunner-ninja/kustomization.yaml | 25 +++ .../kustomization.yaml | 25 +++ .../default-immich-crack/kustomization.yaml | 25 +++ .../kustomization.yaml | 25 +++ .../kustomization.yaml | 25 +++ .../default-watschn-studio/kustomization.yaml | 25 +++ .../cephfs/garrysmod/kustomization.yaml | 25 +++ apps/backup/cephfs/gramps/kustomization.yaml | 25 +++ apps/backup/cephfs/loops/kustomization.yaml | 25 +++ .../cephfs/paperless/kustomization.yaml | 25 +++ .../backup/cephfs/pixelfed/kustomization.yaml | 25 +++ apps/backup/kustomization.yaml | 30 +++ .../postgresql/asciinema/kustomization.yaml | 27 +++ .../postgresql/authentik/kustomization.yaml | 24 +++ .../backup/postgresql/base/kustomization.yaml | 4 + apps/backup/postgresql/base/resources.yaml | 106 ++++++++++ .../default-mumble/kustomization.yaml | 24 +++ .../default-shared/kustomization.yaml | 24 +++ .../postgresql/ghostfolio/kustomization.yaml | 24 +++ .../postgresql/grafana/kustomization.yaml | 29 +++ .../postgresql/gramps/kustomization.yaml | 24 +++ .../homeassistant/kustomization.yaml | 29 +++ .../postgresql/immich/kustomization.yaml | 25 +++ .../postgresql/mastodon/kustomization.yaml | 30 +++ .../postgresql/matrix/kustomization.yaml | 24 +++ .../postgresql/mcserver/kustomization.yaml | 24 +++ .../postgresql/nextcloud/kustomization.yaml | 24 +++ .../backup/postgresql/ombi/kustomization.yaml | 29 +++ .../postgresql/paperless/kustomization.yaml | 29 +++ .../backup/postgresql/quay/kustomization.yaml | 30 +++ .../postgresql/wiki-js/kustomization.yaml | 29 +++ apps/backup/s3/kustomization.yaml | 4 + apps/backup/s3/resources.yaml | 185 ++++++++++++++++++ apps/prometheus/resources/backup-alerts.yaml | 19 +- bootstrap/backup-project.yaml | 40 ++++ 40 files changed, 1229 insertions(+), 12 deletions(-) create mode 100644 apps/backup/cephfs/base/kustomization.yaml create mode 100644 apps/backup/cephfs/base/resources.yaml create mode 100644 apps/backup/cephfs/default-brgkepler-com/kustomization.yaml create mode 100644 apps/backup/cephfs/default-brunner-ninja/kustomization.yaml create mode 100644 apps/backup/cephfs/default-gschissenes-equipment/kustomization.yaml create mode 100644 apps/backup/cephfs/default-immich-crack/kustomization.yaml create mode 100644 apps/backup/cephfs/default-immich-sync-stats/kustomization.yaml create mode 100644 apps/backup/cephfs/default-pfostendynamics/kustomization.yaml create mode 100644 apps/backup/cephfs/default-watschn-studio/kustomization.yaml create mode 100644 apps/backup/cephfs/garrysmod/kustomization.yaml create mode 100644 apps/backup/cephfs/gramps/kustomization.yaml create mode 100644 apps/backup/cephfs/loops/kustomization.yaml create mode 100644 apps/backup/cephfs/paperless/kustomization.yaml create mode 100644 apps/backup/cephfs/pixelfed/kustomization.yaml create mode 100644 apps/backup/postgresql/asciinema/kustomization.yaml create mode 100644 apps/backup/postgresql/authentik/kustomization.yaml create mode 100644 apps/backup/postgresql/base/kustomization.yaml create mode 100644 apps/backup/postgresql/base/resources.yaml create mode 100644 apps/backup/postgresql/default-mumble/kustomization.yaml create mode 100644 apps/backup/postgresql/default-shared/kustomization.yaml create mode 100644 apps/backup/postgresql/ghostfolio/kustomization.yaml create mode 100644 apps/backup/postgresql/grafana/kustomization.yaml create mode 100644 apps/backup/postgresql/gramps/kustomization.yaml create mode 100644 apps/backup/postgresql/homeassistant/kustomization.yaml create mode 100644 apps/backup/postgresql/immich/kustomization.yaml create mode 100644 apps/backup/postgresql/mastodon/kustomization.yaml create mode 100644 apps/backup/postgresql/matrix/kustomization.yaml create mode 100644 apps/backup/postgresql/mcserver/kustomization.yaml create mode 100644 apps/backup/postgresql/nextcloud/kustomization.yaml create mode 100644 apps/backup/postgresql/ombi/kustomization.yaml create mode 100644 apps/backup/postgresql/paperless/kustomization.yaml create mode 100644 apps/backup/postgresql/quay/kustomization.yaml create mode 100644 apps/backup/postgresql/wiki-js/kustomization.yaml create mode 100644 apps/backup/s3/kustomization.yaml create mode 100644 apps/backup/s3/resources.yaml diff --git a/apps/backup/README.md b/apps/backup/README.md index 689f794..d174b41 100644 --- a/apps/backup/README.md +++ b/apps/backup/README.md @@ -1,7 +1,21 @@ # Cluster backups -The first backup schedule dumps every database and the global roles from the host PostgreSQL server on arschrock. It uses the host Unix socket and peer authentication; no database password or fixed ClusterIP is stored in Git. The pod is pinned to arschrock and checks that `/mnt/backup` is on bcachefs before creating output. Kubernetes uses `hostPath.type: Directory`, never `DirectoryOrCreate`. +All jobs run on arschrock and write beneath `/mnt/backup/kubernetes`. Each job uses `hostPath.type: Directory` and checks that its mounted destination is bcachefs before writing. Database and object store credentials are mounted from their existing namespace Secrets. The public GitOps repository contains no credential values. -One-time host preparation: `sudo install -d -o 962 -g 962 -m 0700 /mnt/backup/kubernetes/postgresql/arschrock`. Verify UID 962 still belongs to the host `postgres` user before changing it. Existing files directly under `/mnt/backup` are untouched. Each run writes a timestamped directory with `globals.sql`, one custom-format dump per database, and a `complete` marker written only after all dumps validate with `pg_restore --list`. A directory without `complete` is incomplete and must not be used for restore. No automatic retention or deletion is enabled yet. +## PostgreSQL -The Prometheus rules in `apps/prometheus/resources/backup-alerts.yaml` alert on failed jobs and a missing or stale successful host PostgreSQL backup. This is **not yet a complete cluster backup**: in-cluster PostgreSQL, Valkey, CephFS, and RGW/S3 still need schedules, credential handling, and restore tests. etcd and RBD were explicitly deferred. The old scripts and files in `/mnt/backup` are intentionally preserved. +The host job uses the local PostgreSQL Unix socket and peer authentication. In-cluster PostgreSQL jobs run in their database namespace and use that server's service DNS name and existing Secret. Servers with a superuser credential get a globals dump and custom-format dump of every connectable database; instances with app-only credentials dump the configured application database. Dumps are written under a timestamped directory. Each database archive is checked with `pg_restore --list`, and the `complete` marker is written only after the full run succeeds. The globals dump omits role password hashes; restore must reset database passwords from the current Secrets. A directory without `complete` is incomplete and must not be used for restore. + +One-time host preparation for PostgreSQL: `sudo install -d -o 962 -g 962 -m 0700 /mnt/backup/kubernetes/postgresql/arschrock`. UID 962 is the host `postgres` account. The PostgreSQL backups are not automatically pruned. + +## CephFS and S3 + +Each CephFS CronJob mounts one existing CephFS PVC read-only and mirrors it to its own pre-created bcachefs directory with `rsync --archive --numeric-ids --inplace --partial --delete`. It refuses to run unless `/source` is a CephFS mount and `/backup` is bcachefs. These are mirrors, so source deletions and changes propagate to the mirror; they do not provide historical snapshots. + +The RGW job syncs all 12 current buckets with `rclone sync`. It reads Rook-generated per-bucket object user Secrets in `rook-ceph`; the `nextcloud-new` bucket uses the existing `nextcloud` object user. The destination is a local mirror, and remote deletions propagate. The first complete CephFS and S3 runs are needed before declaring these data sets backed up. + +## Schedules and alerts + +PostgreSQL runs daily between 02:15 and 06:30 Europe/Vienna; CephFS mirrors run between 07:00 and 12:30; RGW starts at 13:00. The schedule staggers starts, but a long run can overlap a later one. Prometheus alerts on failed backup Jobs and any backup CronJob without a success in 25 hours. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory. + +Valkey/Redis persistence is not included yet. etcd and RBD remain deferred. Existing files and old scripts directly under `/mnt/backup` are preserved. No restore has been performed; the PostgreSQL archive checks validate archive readability, not a full restore rehearsal. diff --git a/apps/backup/arschrock-postgresql.yaml b/apps/backup/arschrock-postgresql.yaml index 1c5c8da..b78e6b2 100644 --- a/apps/backup/arschrock-postgresql.yaml +++ b/apps/backup/arschrock-postgresql.yaml @@ -14,7 +14,7 @@ data: export PGHOST=/postgresql PGUSER=postgres PGDATABASE=postgres destination="/backup/kubernetes/postgresql/arschrock/$(date -u +%Y-%m-%dT%H-%M-%SZ)" mkdir -m 0700 "$destination" - pg_dumpall --globals-only > "$destination/globals.sql.partial" + pg_dumpall --globals-only --no-role-passwords > "$destination/globals.sql.partial" test -s "$destination/globals.sql.partial" mv "$destination/globals.sql.partial" "$destination/globals.sql" psql -AtX -v ON_ERROR_STOP=1 -c 'SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate ORDER BY datname' > "$destination/databases.list" diff --git a/apps/backup/cephfs/base/kustomization.yaml b/apps/backup/cephfs/base/kustomization.yaml new file mode 100644 index 0000000..4c2c9a2 --- /dev/null +++ b/apps/backup/cephfs/base/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - resources.yaml diff --git a/apps/backup/cephfs/base/resources.yaml b/apps/backup/cephfs/base/resources.yaml new file mode 100644 index 0000000..8c9bd8a --- /dev/null +++ b/apps/backup/cephfs/base/resources.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cephfs-backup-script +data: + backup.sh: | + #!/bin/sh + set -eu + test "$(stat -f -c %T /backup)" = bcachefs + test "$(stat -f -c %T /source)" = ceph + test -d /source + rsync --archive --numeric-ids --inplace --partial --delete --human-readable --stats /source/ /backup/ + echo "Completed CephFS mirror for $BACKUP_TARGET" +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cephfs-backup +spec: + schedule: "0 3 * * *" + timeZone: Europe/Vienna + concurrencyPolicy: Forbid + startingDeadlineSeconds: 600 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 7 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 21600 + template: + spec: + nodeSelector: + kubernetes.io/hostname: arschrock + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: rsync + image: servercontainers/rsync:a3.24.1-r3.4.1 + imagePullPolicy: IfNotPresent + command: [/bin/sh, /scripts/backup.sh] + env: + - name: BACKUP_TARGET + value: target + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + add: [CHOWN, DAC_OVERRIDE, FOWNER] + volumeMounts: + - name: backup + mountPath: /backup + - name: source + mountPath: /source + readOnly: true + - name: scripts + mountPath: /scripts + readOnly: true + volumes: + - name: backup + hostPath: + path: /mnt/backup/kubernetes/cephfs/target + type: Directory + - name: source + persistentVolumeClaim: + claimName: target + readOnly: true + - name: scripts + configMap: + name: cephfs-backup-script + defaultMode: 0555 diff --git a/apps/backup/cephfs/default-brgkepler-com/kustomization.yaml b/apps/backup/cephfs/default-brgkepler-com/kustomization.yaml new file mode 100644 index 0000000..39f1755 --- /dev/null +++ b/apps/backup/cephfs/default-brgkepler-com/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: cephfs-brgkepler- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "0 7 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: default-brgkepler-com}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/default-brgkepler-com, type: Directory} + - name: source + persistentVolumeClaim: {claimName: brgkepler-com, readOnly: true} diff --git a/apps/backup/cephfs/default-brunner-ninja/kustomization.yaml b/apps/backup/cephfs/default-brunner-ninja/kustomization.yaml new file mode 100644 index 0000000..046fa4e --- /dev/null +++ b/apps/backup/cephfs/default-brunner-ninja/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: cephfs-brunner- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "30 7 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: default-brunner-ninja}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/default-brunner-ninja, type: Directory} + - name: source + persistentVolumeClaim: {claimName: brunner-ninja, readOnly: true} diff --git a/apps/backup/cephfs/default-gschissenes-equipment/kustomization.yaml b/apps/backup/cephfs/default-gschissenes-equipment/kustomization.yaml new file mode 100644 index 0000000..dc6a6a1 --- /dev/null +++ b/apps/backup/cephfs/default-gschissenes-equipment/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: cephfs-gschissenes- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "0 8 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: default-gschissenes-equipment}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/default-gschissenes-equipment, type: Directory} + - name: source + persistentVolumeClaim: {claimName: gschissenes-equipment, readOnly: true} diff --git a/apps/backup/cephfs/default-immich-crack/kustomization.yaml b/apps/backup/cephfs/default-immich-crack/kustomization.yaml new file mode 100644 index 0000000..2166f41 --- /dev/null +++ b/apps/backup/cephfs/default-immich-crack/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: cephfs-immich-crack- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "30 8 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: default-immich-crack}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/default-immich-crack, type: Directory} + - name: source + persistentVolumeClaim: {claimName: immich-crack, readOnly: true} diff --git a/apps/backup/cephfs/default-immich-sync-stats/kustomization.yaml b/apps/backup/cephfs/default-immich-sync-stats/kustomization.yaml new file mode 100644 index 0000000..b47a8ae --- /dev/null +++ b/apps/backup/cephfs/default-immich-sync-stats/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: cephfs-immich-sync-stats- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "0 9 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: default-immich-sync-stats}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/default-immich-sync-stats, type: Directory} + - name: source + persistentVolumeClaim: {claimName: immich-sync-stats, readOnly: true} diff --git a/apps/backup/cephfs/default-pfostendynamics/kustomization.yaml b/apps/backup/cephfs/default-pfostendynamics/kustomization.yaml new file mode 100644 index 0000000..f429d2b --- /dev/null +++ b/apps/backup/cephfs/default-pfostendynamics/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: cephfs-pfostendynamics- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "30 9 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: default-pfostendynamics-com}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/default-pfostendynamics-com, type: Directory} + - name: source + persistentVolumeClaim: {claimName: pfostendynamics-com, readOnly: true} diff --git a/apps/backup/cephfs/default-watschn-studio/kustomization.yaml b/apps/backup/cephfs/default-watschn-studio/kustomization.yaml new file mode 100644 index 0000000..f39afe1 --- /dev/null +++ b/apps/backup/cephfs/default-watschn-studio/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: cephfs-watschn- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "0 10 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: default-watschn-studio}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/default-watschn-studio, type: Directory} + - name: source + persistentVolumeClaim: {claimName: watschn-studio, readOnly: true} diff --git a/apps/backup/cephfs/garrysmod/kustomization.yaml b/apps/backup/cephfs/garrysmod/kustomization.yaml new file mode 100644 index 0000000..32d76b2 --- /dev/null +++ b/apps/backup/cephfs/garrysmod/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: garrysmod +namePrefix: cephfs-garrysmod- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "30 10 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: garrysmod-garrysmod-steam}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/garrysmod-garrysmod-steam, type: Directory} + - name: source + persistentVolumeClaim: {claimName: garrysmod-steam, readOnly: true} diff --git a/apps/backup/cephfs/gramps/kustomization.yaml b/apps/backup/cephfs/gramps/kustomization.yaml new file mode 100644 index 0000000..d26e020 --- /dev/null +++ b/apps/backup/cephfs/gramps/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: gramps +namePrefix: cephfs-gramps- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "0 11 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: gramps-grampsweb-data}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/gramps-grampsweb-data, type: Directory} + - name: source + persistentVolumeClaim: {claimName: grampsweb-data, readOnly: true} diff --git a/apps/backup/cephfs/loops/kustomization.yaml b/apps/backup/cephfs/loops/kustomization.yaml new file mode 100644 index 0000000..82ee30a --- /dev/null +++ b/apps/backup/cephfs/loops/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: loops +namePrefix: cephfs-loops- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "30 11 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: loops-app-data}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/loops-app-data, type: Directory} + - name: source + persistentVolumeClaim: {claimName: app-data, readOnly: true} diff --git a/apps/backup/cephfs/paperless/kustomization.yaml b/apps/backup/cephfs/paperless/kustomization.yaml new file mode 100644 index 0000000..bc48c92 --- /dev/null +++ b/apps/backup/cephfs/paperless/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: paperless-ngx +namePrefix: cephfs-paperless- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "0 12 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: paperless-paperless-data}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/paperless-paperless-data, type: Directory} + - name: source + persistentVolumeClaim: {claimName: paperless-data, readOnly: true} diff --git a/apps/backup/cephfs/pixelfed/kustomization.yaml b/apps/backup/cephfs/pixelfed/kustomization.yaml new file mode 100644 index 0000000..a7e5a60 --- /dev/null +++ b/apps/backup/cephfs/pixelfed/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: pixelfed +namePrefix: cephfs-pixelfed- +resources: [../base] +patches: + - target: {kind: CronJob, name: cephfs-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: cephfs-backup} + spec: + schedule: "30 12 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rsync + env: [{name: BACKUP_TARGET, value: pixelfed-app-data}] + volumes: + - name: backup + hostPath: {path: /mnt/backup/kubernetes/cephfs/pixelfed-app-data, type: Directory} + - name: source + persistentVolumeClaim: {claimName: app-data, readOnly: true} diff --git a/apps/backup/kustomization.yaml b/apps/backup/kustomization.yaml index aa36412..307af06 100644 --- a/apps/backup/kustomization.yaml +++ b/apps/backup/kustomization.yaml @@ -2,3 +2,33 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - arschrock-postgresql.yaml + - postgresql/asciinema + - postgresql/authentik + - postgresql/default-mumble + - postgresql/default-shared + - postgresql/ghostfolio + - postgresql/grafana + - postgresql/gramps + - postgresql/homeassistant + - postgresql/immich + - postgresql/mastodon + - postgresql/matrix + - postgresql/mcserver + - postgresql/nextcloud + - postgresql/ombi + - postgresql/paperless + - postgresql/quay + - postgresql/wiki-js + - cephfs/default-brunner-ninja + - cephfs/default-brgkepler-com + - cephfs/default-gschissenes-equipment + - cephfs/default-immich-crack + - cephfs/default-immich-sync-stats + - cephfs/default-pfostendynamics + - cephfs/default-watschn-studio + - cephfs/garrysmod + - cephfs/gramps + - cephfs/loops + - cephfs/paperless + - cephfs/pixelfed + - s3 diff --git a/apps/backup/postgresql/asciinema/kustomization.yaml b/apps/backup/postgresql/asciinema/kustomization.yaml new file mode 100644 index 0000000..ec33bdc --- /dev/null +++ b/apps/backup/postgresql/asciinema/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: asciinema +namePrefix: asciinema- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "30 2 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: asciinema} + - {name: BACKUP_MODE, value: database} + - {name: PGHOST, value: postgresql} + - {name: PGUSER, value: asciinema} + - {name: PGDATABASE, value: asciinema} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: asciinema-secrets, key: database-password}} diff --git a/apps/backup/postgresql/authentik/kustomization.yaml b/apps/backup/postgresql/authentik/kustomization.yaml new file mode 100644 index 0000000..7bca5db --- /dev/null +++ b/apps/backup/postgresql/authentik/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: authentik +namePrefix: authentik- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "45 2 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: authentik} + - {name: PGHOST, value: authentik-postgresql} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: authentik-postgresql, key: postgres-password}} diff --git a/apps/backup/postgresql/base/kustomization.yaml b/apps/backup/postgresql/base/kustomization.yaml new file mode 100644 index 0000000..4c2c9a2 --- /dev/null +++ b/apps/backup/postgresql/base/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - resources.yaml diff --git a/apps/backup/postgresql/base/resources.yaml b/apps/backup/postgresql/base/resources.yaml new file mode 100644 index 0000000..eca2cec --- /dev/null +++ b/apps/backup/postgresql/base/resources.yaml @@ -0,0 +1,106 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgresql-backup-script +data: + backup.sh: | + #!/bin/sh + set -eu + test "$(stat -f -c %T /backup)" = bcachefs + case "$BACKUP_TARGET" in *[!a-zA-Z0-9_-]*|'') echo "Invalid backup target" >&2; exit 1 ;; esac + PGUSER="${PGUSER:-postgres}" + PGDATABASE="${PGDATABASE:-postgres}" + export PGUSER PGDATABASE + test -n "$PGHOST" -a -n "$PGUSER" -a -n "$PGPASSWORD" + root="/backup/kubernetes/postgresql/$BACKUP_TARGET" + mkdir -p "$root" + run="$root/$(date -u +%Y-%m-%dT%H-%M-%SZ)" + mkdir -m 0700 "$run" + if [ "$BACKUP_MODE" = all ]; then + pg_dumpall --globals-only --no-role-passwords > "$run/globals.sql.partial" + test -s "$run/globals.sql.partial" + mv "$run/globals.sql.partial" "$run/globals.sql" + psql -AtX -v ON_ERROR_STOP=1 -d postgres -c 'SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate ORDER BY datname' > "$run/databases.list" + while IFS= read -r database; do + case "$database" in *[!a-zA-Z0-9_-]*|'') echo "Invalid database name" >&2; exit 1 ;; esac + pg_dump --format=custom --file="$run/$database.dump.partial" --dbname="$database" + pg_restore --list "$run/$database.dump.partial" >/dev/null + mv "$run/$database.dump.partial" "$run/$database.dump" + done < "$run/databases.list" + test "$(find "$run" -maxdepth 1 -name '*.dump' | wc -l)" -gt 0 + else + case "$PGDATABASE" in *[!a-zA-Z0-9_-]*|'') echo "Invalid database name" >&2; exit 1 ;; esac + pg_dump --format=custom --file="$run/$PGDATABASE.dump.partial" --dbname="$PGDATABASE" + pg_restore --list "$run/$PGDATABASE.dump.partial" >/dev/null + mv "$run/$PGDATABASE.dump.partial" "$run/$PGDATABASE.dump" + fi + date -u +%FT%TZ > "$run/complete" + echo "Completed PostgreSQL backup for $BACKUP_TARGET at $run" +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: postgresql-backup +spec: + schedule: "30 2 * * *" + timeZone: Europe/Vienna + concurrencyPolicy: Forbid + startingDeadlineSeconds: 600 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 7 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 21600 + template: + spec: + nodeSelector: + kubernetes.io/hostname: arschrock + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsUser: 962 + runAsGroup: 962 + containers: + - name: pg-dump + image: postgres:18.6 + imagePullPolicy: IfNotPresent + command: [/bin/sh, /scripts/backup.sh] + env: + - name: BACKUP_TARGET + value: target + - name: BACKUP_MODE + value: all + - name: PGHOST + value: postgresql + - name: PGSSLMODE + value: prefer + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: postgres-backup-credentials + key: password + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: backup + mountPath: /backup + - name: scripts + mountPath: /scripts + readOnly: true + volumes: + - name: backup + hostPath: + path: /mnt/backup + type: Directory + - name: scripts + configMap: + name: postgresql-backup-script + defaultMode: 0555 diff --git a/apps/backup/postgresql/default-mumble/kustomization.yaml b/apps/backup/postgresql/default-mumble/kustomization.yaml new file mode 100644 index 0000000..280d556 --- /dev/null +++ b/apps/backup/postgresql/default-mumble/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: default-mumble- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "0 3 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: default-mumble} + - {name: PGHOST, value: mumble-postgresql} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: mumble-postgres, key: postgres-password}} diff --git a/apps/backup/postgresql/default-shared/kustomization.yaml b/apps/backup/postgresql/default-shared/kustomization.yaml new file mode 100644 index 0000000..2078d30 --- /dev/null +++ b/apps/backup/postgresql/default-shared/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +namePrefix: default-shared- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "15 3 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: default-shared} + - {name: PGHOST, value: postgresql} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: postgresql, key: postgres-password}} diff --git a/apps/backup/postgresql/ghostfolio/kustomization.yaml b/apps/backup/postgresql/ghostfolio/kustomization.yaml new file mode 100644 index 0000000..f7e2faa --- /dev/null +++ b/apps/backup/postgresql/ghostfolio/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ghostfolio +namePrefix: ghostfolio- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "30 3 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: ghostfolio} + - {name: PGHOST, value: ghostfolio-postgresql} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: ghostfolio-postgres, key: postgres-password}} diff --git a/apps/backup/postgresql/grafana/kustomization.yaml b/apps/backup/postgresql/grafana/kustomization.yaml new file mode 100644 index 0000000..8f40c9e --- /dev/null +++ b/apps/backup/postgresql/grafana/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: grafana +namePrefix: grafana- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "45 3 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: grafana} + - {name: BACKUP_MODE, value: database} + - {name: PGHOST, value: postgresql} + - name: PGUSER + valueFrom: {secretKeyRef: {name: grafana-postgresql, key: USERDB_USER}} + - name: PGDATABASE + valueFrom: {secretKeyRef: {name: grafana-postgresql, key: USERDB_NAME}} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: grafana-postgresql, key: USERDB_PASSWORD}} diff --git a/apps/backup/postgresql/gramps/kustomization.yaml b/apps/backup/postgresql/gramps/kustomization.yaml new file mode 100644 index 0000000..c6b0bee --- /dev/null +++ b/apps/backup/postgresql/gramps/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: gramps +namePrefix: gramps- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "0 4 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: gramps} + - {name: PGHOST, value: postgresql} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: gramps-postgresql, key: postgres-password}} diff --git a/apps/backup/postgresql/homeassistant/kustomization.yaml b/apps/backup/postgresql/homeassistant/kustomization.yaml new file mode 100644 index 0000000..aa3d00a --- /dev/null +++ b/apps/backup/postgresql/homeassistant/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: homeassistant +namePrefix: homeassistant- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "15 4 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: homeassistant} + - {name: BACKUP_MODE, value: database} + - {name: PGHOST, value: postgresql} + - name: PGUSER + valueFrom: {secretKeyRef: {name: homeassistant-postgresql, key: USERDB_USER}} + - name: PGDATABASE + valueFrom: {secretKeyRef: {name: homeassistant-postgresql, key: USERDB_NAME}} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: homeassistant-postgresql, key: USERDB_PASSWORD}} diff --git a/apps/backup/postgresql/immich/kustomization.yaml b/apps/backup/postgresql/immich/kustomization.yaml new file mode 100644 index 0000000..6982573 --- /dev/null +++ b/apps/backup/postgresql/immich/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: immich +namePrefix: immich- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "30 4 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: immich} + - {name: PGHOST, value: immich-postgresql-rw} + - {name: PGSSLMODE, value: require} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: immich-postgresql-superuser, key: password}} diff --git a/apps/backup/postgresql/mastodon/kustomization.yaml b/apps/backup/postgresql/mastodon/kustomization.yaml new file mode 100644 index 0000000..311f687 --- /dev/null +++ b/apps/backup/postgresql/mastodon/kustomization.yaml @@ -0,0 +1,30 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: mastodon +namePrefix: mastodon- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "45 4 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: mastodon} + - {name: BACKUP_MODE, value: database} + - {name: PGHOST, value: mastodon-postgresql-rw} + - {name: PGSSLMODE, value: require} + - name: PGUSER + valueFrom: {secretKeyRef: {name: mastodon-postgresql-app, key: username}} + - name: PGDATABASE + valueFrom: {secretKeyRef: {name: mastodon-postgresql-app, key: dbname}} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: mastodon-postgresql-app, key: password}} diff --git a/apps/backup/postgresql/matrix/kustomization.yaml b/apps/backup/postgresql/matrix/kustomization.yaml new file mode 100644 index 0000000..8e4f508 --- /dev/null +++ b/apps/backup/postgresql/matrix/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: matrix +namePrefix: matrix- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "0 5 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: matrix} + - {name: PGHOST, value: postgresql} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: postgresql, key: postgres-password}} diff --git a/apps/backup/postgresql/mcserver/kustomization.yaml b/apps/backup/postgresql/mcserver/kustomization.yaml new file mode 100644 index 0000000..584094f --- /dev/null +++ b/apps/backup/postgresql/mcserver/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: mcserver +namePrefix: mcserver- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "15 5 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: mcserver} + - {name: PGHOST, value: paper-postgresql} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: paper-postgres, key: postgres-password}} diff --git a/apps/backup/postgresql/nextcloud/kustomization.yaml b/apps/backup/postgresql/nextcloud/kustomization.yaml new file mode 100644 index 0000000..796d49a --- /dev/null +++ b/apps/backup/postgresql/nextcloud/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: nextcloud +namePrefix: nextcloud- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "30 5 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: nextcloud} + - {name: PGHOST, value: nextcloud-postgresql} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: nextcloud-postgresql, key: postgres-password}} diff --git a/apps/backup/postgresql/ombi/kustomization.yaml b/apps/backup/postgresql/ombi/kustomization.yaml new file mode 100644 index 0000000..2d40b7d --- /dev/null +++ b/apps/backup/postgresql/ombi/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ombi +namePrefix: ombi- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "45 5 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: ombi} + - {name: BACKUP_MODE, value: database} + - {name: PGHOST, value: postgresql} + - name: PGUSER + valueFrom: {secretKeyRef: {name: ombi-postgresql, key: USERDB_USER}} + - name: PGDATABASE + valueFrom: {secretKeyRef: {name: ombi-postgresql, key: USERDB_NAME}} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: ombi-postgresql, key: USERDB_PASSWORD}} diff --git a/apps/backup/postgresql/paperless/kustomization.yaml b/apps/backup/postgresql/paperless/kustomization.yaml new file mode 100644 index 0000000..15c504a --- /dev/null +++ b/apps/backup/postgresql/paperless/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: paperless-ngx +namePrefix: paperless- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "0 6 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: paperless} + - {name: BACKUP_MODE, value: database} + - {name: PGHOST, value: postgresql} + - name: PGUSER + valueFrom: {secretKeyRef: {name: paperless-postgres, key: USERDB_USER}} + - name: PGDATABASE + valueFrom: {secretKeyRef: {name: paperless-postgres, key: USERDB_NAME}} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: paperless-postgres, key: USERDB_PASSWORD}} diff --git a/apps/backup/postgresql/quay/kustomization.yaml b/apps/backup/postgresql/quay/kustomization.yaml new file mode 100644 index 0000000..64cd432 --- /dev/null +++ b/apps/backup/postgresql/quay/kustomization.yaml @@ -0,0 +1,30 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: quay +namePrefix: quay- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "15 6 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: quay} + - {name: BACKUP_MODE, value: database} + - {name: PGHOST, value: quay-postgresql-rw} + - {name: PGSSLMODE, value: require} + - name: PGUSER + valueFrom: {secretKeyRef: {name: quay-postgresql-app, key: username}} + - name: PGDATABASE + valueFrom: {secretKeyRef: {name: quay-postgresql-app, key: dbname}} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: quay-postgresql-app, key: password}} diff --git a/apps/backup/postgresql/wiki-js/kustomization.yaml b/apps/backup/postgresql/wiki-js/kustomization.yaml new file mode 100644 index 0000000..53d3d76 --- /dev/null +++ b/apps/backup/postgresql/wiki-js/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: wiki-js +namePrefix: wiki-js- +resources: [../base] +patches: + - target: {kind: CronJob, name: postgresql-backup} + patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: {name: postgresql-backup} + spec: + schedule: "30 6 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: pg-dump + env: + - {name: BACKUP_TARGET, value: wiki-js} + - {name: BACKUP_MODE, value: database} + - {name: PGHOST, value: postgresql} + - name: PGUSER + valueFrom: {secretKeyRef: {name: wiki-js-postgresql, key: USERDB_USER}} + - name: PGDATABASE + valueFrom: {secretKeyRef: {name: wiki-js-postgresql, key: USERDB_NAME}} + - name: PGPASSWORD + valueFrom: {secretKeyRef: {name: wiki-js-postgresql, key: USERDB_PASSWORD}} diff --git a/apps/backup/s3/kustomization.yaml b/apps/backup/s3/kustomization.yaml new file mode 100644 index 0000000..4c2c9a2 --- /dev/null +++ b/apps/backup/s3/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - resources.yaml diff --git a/apps/backup/s3/resources.yaml b/apps/backup/s3/resources.yaml new file mode 100644 index 0000000..b6d7eed --- /dev/null +++ b/apps/backup/s3/resources.yaml @@ -0,0 +1,185 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: s3-backup-script + namespace: rook-ceph +data: + buckets.list: | + nextcloud nextcloud + wiki-js wiki-js + pixelfed pixelfed + quay quay + influxdb influxdb + asciinema asciinema + nextcloud-new nextcloud + loops loops + synapse synapse + authentik authentik + mastodon mastodon + gramps gramps + backup.sh: | + #!/bin/sh + set -eu + test "$(stat -f -c %T /backup)" = bcachefs + endpoint=http://rook-ceph-rgw-my-store.rook-ceph.svc:80 + failures=0 + while read -r bucket owner; do + test -n "$bucket" || continue + remote="BACKUP_$(printf '%s' "$bucket" | tr '[:lower:]-' '[:upper:]_')" + access_key=$(cat "/run/rgw/$owner/AccessKey") + secret_key=$(cat "/run/rgw/$owner/SecretKey") + test -n "$access_key" -a -n "$secret_key" + export "RCLONE_CONFIG_${remote}_TYPE=s3" + export "RCLONE_CONFIG_${remote}_PROVIDER=Ceph" + export "RCLONE_CONFIG_${remote}_ENDPOINT=$endpoint" + export "RCLONE_CONFIG_${remote}_REGION=" + export "RCLONE_CONFIG_${remote}_ACCESS_KEY_ID=$access_key" + export "RCLONE_CONFIG_${remote}_SECRET_ACCESS_KEY=$secret_key" + unset access_key secret_key + echo "Syncing RGW bucket $bucket" + if ! rclone --config /dev/null sync "$remote:$bucket" "/backup/$bucket" --transfers 2 --checkers 4 --stats 1m; then + echo "RGW backup failed for bucket $bucket" >&2 + failures=$((failures + 1)) + fi + done < /scripts/buckets.list + test "$failures" -eq 0 || exit 1 + echo "Completed all RGW bucket mirrors" +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: s3-backup + namespace: rook-ceph +spec: + schedule: "0 13 * * *" + timeZone: Europe/Vienna + concurrencyPolicy: Forbid + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 7 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 43200 + template: + spec: + nodeSelector: + kubernetes.io/hostname: arschrock + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsUser: 962 + runAsGroup: 962 + containers: + - name: rclone + image: rclone/rclone:1.75.0 + imagePullPolicy: IfNotPresent + command: [/bin/sh, /scripts/backup.sh] + env: + - name: XDG_CONFIG_HOME + value: /tmp/.config + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: backup + mountPath: /backup + - name: scripts + mountPath: /scripts + readOnly: true + - name: rgw-asciinema + mountPath: /run/rgw/asciinema + readOnly: true + - name: rgw-authentik + mountPath: /run/rgw/authentik + readOnly: true + - name: rgw-gramps + mountPath: /run/rgw/gramps + readOnly: true + - name: rgw-influxdb + mountPath: /run/rgw/influxdb + readOnly: true + - name: rgw-loops + mountPath: /run/rgw/loops + readOnly: true + - name: rgw-mastodon + mountPath: /run/rgw/mastodon + readOnly: true + - name: rgw-nextcloud + mountPath: /run/rgw/nextcloud + readOnly: true + - name: rgw-pixelfed + mountPath: /run/rgw/pixelfed + readOnly: true + - name: rgw-quay + mountPath: /run/rgw/quay + readOnly: true + - name: rgw-synapse + mountPath: /run/rgw/synapse + readOnly: true + - name: rgw-wiki-js + mountPath: /run/rgw/wiki-js + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: backup + hostPath: + path: /mnt/backup/kubernetes/s3 + type: Directory + - name: scripts + configMap: + name: s3-backup-script + defaultMode: 0555 + - name: rgw-asciinema + secret: + secretName: rook-ceph-object-user-my-store-asciinema + defaultMode: 0444 + - name: rgw-authentik + secret: + secretName: rook-ceph-object-user-my-store-authentik + defaultMode: 0444 + - name: rgw-gramps + secret: + secretName: rook-ceph-object-user-my-store-gramps + defaultMode: 0444 + - name: rgw-influxdb + secret: + secretName: rook-ceph-object-user-my-store-influxdb + defaultMode: 0444 + - name: rgw-loops + secret: + secretName: rook-ceph-object-user-my-store-loops + defaultMode: 0444 + - name: rgw-mastodon + secret: + secretName: rook-ceph-object-user-my-store-mastodon + defaultMode: 0444 + - name: rgw-nextcloud + secret: + secretName: rook-ceph-object-user-my-store-nextcloud + defaultMode: 0444 + - name: rgw-pixelfed + secret: + secretName: rook-ceph-object-user-my-store-pixelfed + defaultMode: 0444 + - name: rgw-quay + secret: + secretName: rook-ceph-object-user-my-store-quay + defaultMode: 0444 + - name: rgw-synapse + secret: + secretName: rook-ceph-object-user-my-store-synapse + defaultMode: 0444 + - name: rgw-wiki-js + secret: + secretName: rook-ceph-object-user-my-store-wiki-js + defaultMode: 0444 + - name: tmp + emptyDir: {} diff --git a/apps/prometheus/resources/backup-alerts.yaml b/apps/prometheus/resources/backup-alerts.yaml index bdf9340..601f504 100644 --- a/apps/prometheus/resources/backup-alerts.yaml +++ b/apps/prometheus/resources/backup-alerts.yaml @@ -10,26 +10,29 @@ spec: - name: backup-jobs rules: - alert: BackupCronJobFailed - expr: kube_job_status_failed{namespace="backup"} > 0 + expr: |- + (kube_job_status_failed{job_name=~".*backup.*"} > 0) + and on(namespace,job_name) + (time() - kube_job_created{job_name=~".*backup.*"} < 86400) for: 5m labels: severity: critical annotations: summary: "Backup job failed" description: "{{ $labels.namespace }}/{{ $labels.job_name }} failed; inspect its pod logs and /mnt/backup mount." - - alert: ArschrockPostgreSQLBackupStale + - alert: ScheduledBackupStale expr: |- - (time() - kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"} > 90000) + (time() - kube_cronjob_status_last_successful_time{cronjob=~".*backup.*"} > 90000) or ( - (kube_cronjob_info{namespace="backup",cronjob="arschrock-postgresql-backup"} - unless kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"}) + (kube_cronjob_info{cronjob=~".*backup.*"} + unless kube_cronjob_status_last_successful_time{cronjob=~".*backup.*"}) and on(namespace,cronjob) - (time() - kube_cronjob_created{namespace="backup",cronjob="arschrock-postgresql-backup"} > 90000) + (time() - kube_cronjob_created{cronjob=~".*backup.*"} > 90000) ) for: 10m labels: severity: critical annotations: - summary: "Arschrock PostgreSQL backup has no recent successful run" - description: "The daily PostgreSQL dump is older than 25 hours or has never succeeded." + summary: "A scheduled backup has no recent successful run" + description: "{{ $labels.namespace }}/{{ $labels.cronjob }} has not completed successfully in 25 hours." diff --git a/bootstrap/backup-project.yaml b/bootstrap/backup-project.yaml index 750dad3..fca1cc5 100644 --- a/bootstrap/backup-project.yaml +++ b/bootstrap/backup-project.yaml @@ -10,6 +10,46 @@ spec: destinations: - namespace: backup server: https://kubernetes.default.svc + - namespace: asciinema + server: https://kubernetes.default.svc + - namespace: authentik + server: https://kubernetes.default.svc + - namespace: default + server: https://kubernetes.default.svc + - namespace: ghostfolio + server: https://kubernetes.default.svc + - namespace: garrysmod + server: https://kubernetes.default.svc + - namespace: grafana + server: https://kubernetes.default.svc + - namespace: gramps + server: https://kubernetes.default.svc + - namespace: homeassistant + server: https://kubernetes.default.svc + - namespace: immich + server: https://kubernetes.default.svc + - namespace: loops + server: https://kubernetes.default.svc + - namespace: mastodon + server: https://kubernetes.default.svc + - namespace: matrix + server: https://kubernetes.default.svc + - namespace: mcserver + server: https://kubernetes.default.svc + - namespace: nextcloud + server: https://kubernetes.default.svc + - namespace: ombi + server: https://kubernetes.default.svc + - namespace: paperless-ngx + server: https://kubernetes.default.svc + - namespace: pixelfed + server: https://kubernetes.default.svc + - namespace: quay + server: https://kubernetes.default.svc + - namespace: rook-ceph + server: https://kubernetes.default.svc + - namespace: wiki-js + server: https://kubernetes.default.svc namespaceResourceWhitelist: - group: "" kind: ConfigMap