55 lines
2.3 KiB
YAML
55 lines
2.3 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: backup-jobs
|
|
namespace: prometheus
|
|
labels:
|
|
release: prometheus
|
|
spec:
|
|
groups:
|
|
- name: backup-jobs
|
|
rules:
|
|
- alert: BackupCronJobFailed
|
|
expr: |-
|
|
(kube_job_status_failed{job_name=~".*backup.*"} > 0)
|
|
and on(namespace,job_name)
|
|
(time() - kube_job_created{job_name=~".*backup.*"} < 86400)
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Backup job failed"
|
|
description: "{{ $labels.namespace }}/{{ $labels.job_name }} failed; inspect its pod logs and /mnt/backup mount."
|
|
- alert: ScheduledBackupStale
|
|
expr: |-
|
|
(time() - kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000)
|
|
or
|
|
(
|
|
(kube_cronjob_info{cronjob=~".*backup.*",cronjob!="s3-backup"}
|
|
unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{cronjob=~".*backup.*",cronjob!="s3-backup"})
|
|
and on(namespace,cronjob)
|
|
(time() - kube_cronjob_created{cronjob=~".*backup.*",cronjob!="s3-backup"} > 90000)
|
|
)
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "A scheduled backup has no recent successful run"
|
|
description: "{{ $labels.namespace }}/{{ $labels.cronjob }} has not completed successfully in 25 hours."
|
|
- alert: S3BackupStale
|
|
expr: |-
|
|
(time() - kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"} > 172800)
|
|
or
|
|
(
|
|
(kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup"}
|
|
unless on(namespace,cronjob) kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"})
|
|
and on(namespace,cronjob)
|
|
(time() - kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"} > 172800)
|
|
)
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "The S3 backup has no recent successful run"
|
|
description: "rook-ceph/s3-backup has not completed successfully in 48 hours."
|