diff --git a/apps/prometheus/resources/backup-alerts.yaml b/apps/prometheus/resources/backup-alerts.yaml index 3403998..bdf9340 100644 --- a/apps/prometheus/resources/backup-alerts.yaml +++ b/apps/prometheus/resources/backup-alerts.yaml @@ -18,7 +18,15 @@ spec: summary: "Backup job failed" description: "{{ $labels.namespace }}/{{ $labels.job_name }} failed; inspect its pod logs and /mnt/backup mount." - alert: ArschrockPostgreSQLBackupStale - expr: (time() - kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"} > 90000) or (kube_cronjob_info{namespace="backup",cronjob="arschrock-postgresql-backup"} unless kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"}) + expr: |- + (time() - kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"} > 90000) + or + ( + (kube_cronjob_info{namespace="backup",cronjob="arschrock-postgresql-backup"} + unless kube_cronjob_status_last_successful_time{namespace="backup",cronjob="arschrock-postgresql-backup"}) + and on(namespace,cronjob) + (time() - kube_cronjob_created{namespace="backup",cronjob="arschrock-postgresql-backup"} > 90000) + ) for: 10m labels: severity: critical