From 8521286ce305c66069f7034b97ea58bc9e18d00b Mon Sep 17 00:00:00 2001 From: Matej Feder Date: Mon, 17 Jun 2024 09:18:07 +0200 Subject: [PATCH] monitoring: fix CephPoolGrowthWarning expression Prometheus found duplicate series for the match group `pool_id` and `instance` when evaluating the CephPoolGrowthWarning alert expression. This alert has an evaluation interval of 2 days and did not consider the changing POD names during the Rook migration process. This commit adds the `pod` label to the set of considered labels. Signed-off-by: Matej Feder (cherry picked from commit 6ba084f35accd48d6e1c3ae4440e75d37004231f) --- deploy/charts/rook-ceph-cluster/prometheus/localrules.yaml | 2 +- deploy/examples/monitoring/localrules.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/charts/rook-ceph-cluster/prometheus/localrules.yaml b/deploy/charts/rook-ceph-cluster/prometheus/localrules.yaml index a8ed7cf74..77b00e2b6 100644 --- a/deploy/charts/rook-ceph-cluster/prometheus/localrules.yaml +++ b/deploy/charts/rook-ceph-cluster/prometheus/localrules.yaml @@ -533,7 +533,7 @@ groups: annotations: description: "Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours." summary: "Pool growth rate may soon exceed capacity" - expr: "(predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id, instance) group_right() ceph_pool_metadata) >= 95" + expr: "(predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id, instance, pod) group_right() ceph_pool_metadata) >= 95" labels: oid: "1.3.6.1.4.1.50495.1.2.1.9.2" severity: "warning" diff --git a/deploy/examples/monitoring/localrules.yaml b/deploy/examples/monitoring/localrules.yaml index dbef18559..9a976e398 100644 --- a/deploy/examples/monitoring/localrules.yaml +++ b/deploy/examples/monitoring/localrules.yaml @@ -537,7 +537,7 @@ spec: annotations: description: "Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours." summary: "Pool growth rate may soon exceed capacity" - expr: "(predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id, instance) group_right() ceph_pool_metadata) >= 95" + expr: "(predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id, instance, pod) group_right() ceph_pool_metadata) >= 95" labels: oid: "1.3.6.1.4.1.50495.1.2.1.9.2" severity: "warning"