diff --git a/apps/backup/README.md b/apps/backup/README.md index d174b41..4665683 100644 --- a/apps/backup/README.md +++ b/apps/backup/README.md @@ -12,10 +12,10 @@ One-time host preparation for PostgreSQL: `sudo install -d -o 962 -g 962 -m 0700 Each CephFS CronJob mounts one existing CephFS PVC read-only and mirrors it to its own pre-created bcachefs directory with `rsync --archive --numeric-ids --inplace --partial --delete`. It refuses to run unless `/source` is a CephFS mount and `/backup` is bcachefs. These are mirrors, so source deletions and changes propagate to the mirror; they do not provide historical snapshots. -The RGW job syncs all 12 current buckets with `rclone sync`. It reads Rook-generated per-bucket object user Secrets in `rook-ceph`; the `nextcloud-new` bucket uses the existing `nextcloud` object user. The destination is a local mirror, and remote deletions propagate. The first complete CephFS and S3 runs are needed before declaring these data sets backed up. +The RGW job syncs all 12 current buckets with `rclone sync`. It reads Rook-generated per-bucket object user Secrets in `rook-ceph`; the `nextcloud-new` bucket uses the existing `nextcloud` object user. The destination is a local mirror, and remote deletions propagate. The first complete CephFS and S3 runs are needed before declaring these data sets backed up. The combined S3 run has a 48-hour deadline because the buckets total roughly 527 GB; its CronJob forbids overlapping runs. ## Schedules and alerts -PostgreSQL runs daily between 02:15 and 06:30 Europe/Vienna; CephFS mirrors run between 07:00 and 12:30; RGW starts at 13:00. The schedule staggers starts, but a long run can overlap a later one. Prometheus alerts on failed backup Jobs and any backup CronJob without a success in 25 hours. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory. +PostgreSQL runs daily between 02:15 and 06:30 Europe/Vienna; CephFS mirrors run between 07:00 and 12:30; RGW starts at 13:00. CronJobs forbid overlapping runs of the same backup. Prometheus alerts on failed backup Jobs, daily backup CronJobs without a success in 25 hours, and the S3 mirror without a success in 48 hours. A missing `/mnt/backup` mount fails the pod setup or the in-container filesystem check and therefore cannot write into the underlying directory. Valkey/Redis persistence is not included yet. etcd and RBD remain deferred. Existing files and old scripts directly under `/mnt/backup` are preserved. No restore has been performed; the PostgreSQL archive checks validate archive readability, not a full restore rehearsal. diff --git a/apps/backup/s3/resources.yaml b/apps/backup/s3/resources.yaml index 305326b..5a65e9d 100644 --- a/apps/backup/s3/resources.yaml +++ b/apps/backup/s3/resources.yaml @@ -62,7 +62,7 @@ spec: jobTemplate: spec: backoffLimit: 0 - activeDeadlineSeconds: 43200 + activeDeadlineSeconds: 172800 template: spec: nodeSelector: diff --git a/apps/prometheus/resources/backup-alerts.yaml b/apps/prometheus/resources/backup-alerts.yaml index 601f504..64d542e 100644 --- a/apps/prometheus/resources/backup-alerts.yaml +++ b/apps/prometheus/resources/backup-alerts.yaml @@ -22,13 +22,13 @@ spec: description: "{{ $labels.namespace }}/{{ $labels.job_name }} failed; inspect its pod logs and /mnt/backup mount." - alert: ScheduledBackupStale expr: |- - (time() - kube_cronjob_status_last_successful_time{cronjob=~".*backup.*"} > 90000) + (time() - kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000) or ( - (kube_cronjob_info{cronjob=~".*backup.*"} - unless kube_cronjob_status_last_successful_time{cronjob=~".*backup.*"}) + (kube_cronjob_info{cronjob=~".*backup.*",cronjob!="s3-backup"} + unless kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"}) and on(namespace,cronjob) - (time() - kube_cronjob_created{cronjob=~".*backup.*"} > 90000) + (time() - kube_cronjob_created{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000) ) for: 10m labels: @@ -36,3 +36,19 @@ spec: annotations: summary: "A scheduled backup has no recent successful run" description: "{{ $labels.namespace }}/{{ $labels.cronjob }} has not completed successfully in 25 hours." + - alert: S3BackupStale + expr: |- + (time() - kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"} > 172800) + or + ( + (kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup"} + unless kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"}) + and on(namespace,cronjob) + (time() - kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"} > 172800) + ) + for: 10m + labels: + severity: critical + annotations: + summary: "The S3 backup has no recent successful run" + description: "rook-ceph/s3-backup has not completed successfully in 48 hours."