Files
homelab-gitops/tests/alert-recovery.test.yaml
T

85 lines
4.3 KiB
YAML

# Rule files are generated from the PrometheusRule manifests by the test runner.
rule_files:
- backup.rules.yaml
- cpu.rules.yaml
evaluation_interval: 1m
tests:
- name: Successful backups match CronJob info despite its extra schedule labels
interval: 1m
input_series:
- series: 'kube_cronjob_info{namespace="test",cronjob="postgresql-backup",schedule="0 2 * * *",concurrency_policy="Forbid"}'
values: '1+0x4320'
- series: 'kube_cronjob_created{namespace="test",cronjob="postgresql-backup"}'
values: '0+0x4320'
- series: 'kube_cronjob_status_last_successful_time{namespace="test",cronjob="postgresql-backup"}'
values: '255600+0x4320'
- series: 'kube_cronjob_info{namespace="rook-ceph",cronjob="s3-backup",schedule="0 16 * * *",concurrency_policy="Forbid"}'
values: '1+0x4320'
- series: 'kube_cronjob_created{namespace="rook-ceph",cronjob="s3-backup"}'
values: '0+0x4320'
- series: 'kube_cronjob_status_last_successful_time{namespace="rook-ceph",cronjob="s3-backup"}'
values: '255600+0x4320'
promql_expr_test:
- expr: 'count(ALERTS{alertstate="firing",alertname=~"ScheduledBackupStale|S3BackupStale"}) or vector(0)'
eval_time: 72h
exp_samples:
- labels: '{}'
value: 0
- name: Stale and never successful backups still alert
interval: 1m
input_series:
- series: 'kube_cronjob_info{namespace="test",cronjob="never-backup",schedule="0 2 * * *"}'
values: '1+0x1560'
- series: 'kube_cronjob_created{namespace="test",cronjob="never-backup"}'
values: '0+0x1560'
- series: 'kube_cronjob_status_last_successful_time{namespace="test",cronjob="stale-backup"}'
values: '0+0x1560'
promql_expr_test:
- expr: 'count(ALERTS{alertstate="firing",alertname="ScheduledBackupStale"})'
eval_time: 26h
exp_samples:
- labels: '{}'
value: 2
- name: Capacity accounts for control-plane availability
interval: 1m
input_series:
- series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="nonha",namespace="test"}'
values: '27+0x20'
- series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="overfull",namespace="test"}'
values: '49+0x20'
- series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="ha",namespace="test"}'
values: '27+0x20'
- series: 'kube_node_status_allocatable{cluster="nonha",node="large",job="kube-state-metrics",resource="cpu"}'
values: '24+0x20'
- series: 'kube_node_status_allocatable{cluster="nonha",node="others",job="kube-state-metrics",resource="cpu"}'
values: '24+0x20'
- series: 'kube_node_status_allocatable{cluster="overfull",node="large",job="kube-state-metrics",resource="cpu"}'
values: '24+0x20'
- series: 'kube_node_status_allocatable{cluster="overfull",node="others",job="kube-state-metrics",resource="cpu"}'
values: '24+0x20'
- series: 'kube_node_status_allocatable{cluster="ha",node="large",job="kube-state-metrics",resource="cpu"}'
values: '24+0x20'
- series: 'kube_node_status_allocatable{cluster="ha",node="others",job="kube-state-metrics",resource="cpu"}'
values: '24+0x20'
- series: 'kube_node_role{cluster="nonha",node="a",job="kube-state-metrics",role="control-plane"}'
values: '1+0x20'
- series: 'kube_node_role{cluster="overfull",node="a",job="kube-state-metrics",role="control-plane"}'
values: '1+0x20'
- series: 'kube_node_role{cluster="ha",node="a",job="kube-state-metrics",role="control-plane"}'
values: '1+0x20'
- series: 'kube_node_role{cluster="ha",node="b",job="kube-state-metrics",role="control-plane"}'
values: '1+0x20'
- series: 'kube_node_role{cluster="ha",node="c",job="kube-state-metrics",role="control-plane"}'
values: '1+0x20'
promql_expr_test:
- expr: 'count(ALERTS{alertname="KubeCPUOvercommit",alertstate="firing",cluster="nonha"}) or vector(0)'
eval_time: 20m
exp_samples:
- labels: '{}'
value: 0
- expr: 'count(ALERTS{alertname="KubeCPUOvercommit",alertstate="firing",cluster=~"ha|overfull"})'
eval_time: 20m
exp_samples:
- labels: '{}'
value: 2