From 624ca40817026b26c274069ea78aae1a16589fcf Mon Sep 17 00:00:00 2001
From: Anmol Sachan <anmol13694@gmail.com>
Date: Mon, 22 Jun 2020 21:08:42 +0530
Subject: [PATCH] ceph: added PVC utilization alerts

Added warning and critical alerts for all the PVC's created by rook-ceph provisioners. This is required to notify a admin timely to check on PVC utilization and take a required action to prevent workloads from failing.

Closes: BZ1849723
Signed-off-by: Anmol Sachan <anmol13694@gmail.com>
---
 .../monitoring/prometheus-ceph-v14-rules.yaml | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml b/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml
index 8d4490e19..57464e564 100644
--- a/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml
+++ b/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml
@@ -184,6 +184,34 @@ spec:
       for: 1h
       labels:
         severity: warning
+  - name: persistent-volume-alert.rules
+    rules:
+    - alert: PersistentVolumeUsageNearFull
+      annotations:
+        description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed
+          75%. Free up some space.
+        message: PVC {{ $labels.persistentvolumeclaim }} is nearing full. Data deletion
+          is required.
+        severity_level: warning
+        storage_type: ceph
+      expr: |
+        (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass)  group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass)  group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.75
+      for: 5s
+      labels:
+        severity: warning
+    - alert: PersistentVolumeUsageCritical
+      annotations:
+        description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed
+          85%. Free up some space immediately.
+        message: PVC {{ $labels.persistentvolumeclaim }} is critically full. Data
+          deletion is required.
+        severity_level: error
+        storage_type: ceph
+      expr: |
+        (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass)  group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass)  group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.85
+      for: 5s
+      labels:
+        severity: critical
   - name: cluster-state-alert.rules
     rules:
     - alert: CephClusterErrorState