ceph: added prometheus rule file for alerting on external cluster

Signed-off-by: Anmol Sachan <anmol13694@gmail.com>

cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules-external.yaml

@@ -0,0 +1,38 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    prometheus: rook-prometheus
+    role: alert-rules
+  name: prometheus-ceph-rules
+  namespace: rook-ceph
+spec:
+  groups:
+  - name: persistent-volume-alert.rules
+    rules:
+    - alert: PersistentVolumeUsageNearFull
+      annotations:
+        description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed
+          75%. Free up some space.
+        message: PVC {{ $labels.persistentvolumeclaim }} is nearing full. Data deletion
+          is required.
+        severity_level: warning
+        storage_type: ceph
+      expr: |
+        (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass)  group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass)  group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.75
+      for: 5s
+      labels:
+        severity: warning
+    - alert: PersistentVolumeUsageCritical
+      annotations:
+        description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed
+          85%. Free up some space immediately.
+        message: PVC {{ $labels.persistentvolumeclaim }} is critically full. Data
+          deletion is required.
+        severity_level: error
+        storage_type: ceph
+      expr: |
+        (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass)  group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass)  group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.85
+      for: 5s
+      labels:
+        severity: critical