From 136231a71f7f89b5a883ca6a2c6a7bb181ba9db8 Mon Sep 17 00:00:00 2001 From: Anmol Sachan Date: Fri, 19 Feb 2021 18:25:40 +0530 Subject: [PATCH] ceph: fixed prometheus query to avoid many-to-many match error Ceph_daemon label includes references to a replaced OSD that cause a prometheus ruleset to fail due many-to-many match error. The changes to this PR filters the extra matching ceph_daemon time-series. Signed-off-by: Anmol Sachan --- .../kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml b/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml index d328dd905..89d7d4733 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml @@ -14,7 +14,7 @@ spec: kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"} * on (node) group_right() max(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"},"node","$1","exported_instance","(.*)")) by (node) record: cluster:ceph_node_down:join_kube - expr: | - avg(max by(instance) (label_replace(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"}, "instance", "$1", "exported_instance", "(.*)"), "device", "$1", "device", "/dev/(.*)") * on(instance, device) group_right() (irate(node_disk_read_time_seconds_total[1m]) + irate(node_disk_write_time_seconds_total[1m]) / (clamp_min(irate(node_disk_reads_completed_total[1m]), 1) + irate(node_disk_writes_completed_total[1m]))))) + avg(topk by (ceph_daemon) (1, label_replace(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"}, "instance", "$1", "exported_instance", "(.*)"), "device", "$1", "device", "/dev/(.*)")) * on(instance, device) group_right(ceph_daemon) topk by (instance,device) (1,(irate(node_disk_read_time_seconds_total[1m]) + irate(node_disk_write_time_seconds_total[1m]) / (clamp_min(irate(node_disk_reads_completed_total[1m]), 1) + irate(node_disk_writes_completed_total[1m]))))) record: cluster:ceph_disk_latency:join_ceph_node_disk_irate1m - name: telemeter.rules rules: