helm(rook-ceph-cluster): allow prometheusrule edits

This change allows the chart consumer to optionally
edit PrometheusRule CR by either disabling individual
alerts or tweaking their configuration.

Signed-off-by: Artyom Babiy <artyom.babiy@gmail.com>

Documentation/Helm-Charts/ceph-cluster-chart.md

@@ -74,6 +74,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `monitoring.metricsDisabled` | Whether to disable the metrics reported by Ceph. If false, the prometheus mgr module and Ceph exporter are enabled | `false` |
 | `monitoring.prometheusRule.annotations` | Annotations applied to PrometheusRule | `{}` |
 | `monitoring.prometheusRule.labels` | Labels applied to PrometheusRule | `{}` |
+| `monitoring.prometheusRuleOverrides` | Edit Prometheus rules for Ceph alerts | `{}` |
 | `monitoring.rulesNamespaceOverride` | The namespace in which to create the prometheus rules, if different from the rook cluster namespace. If you have multiple rook-ceph clusters in the same k8s cluster, choose the same namespace (ideally, namespace with prometheus deployed) to set rulesNamespaceOverride for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions. | `nil` |
 | `operatorNamespace` | Namespace of the main rook operator | `"rook-ceph"` |
 | `pspEnable` | Create & use PSP resources. Set this to the same value as the rook-ceph chart. | `false` |

deploy/charts/rook-ceph-cluster/templates/prometheusrules.yaml

@@ -1,5 +1,4 @@
-{{- if .Values.monitoring }}
-{{- if .Values.monitoring.createPrometheusRules }}
+{{- if and .Values.monitoring .Values.monitoring.createPrometheusRules }}
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
@@ -17,24 +16,30 @@ metadata:
   name: prometheus-ceph-rules
   namespace: {{ default .Release.Namespace .Values.monitoring.rulesNamespaceOverride }}
 spec:
-# Import the raw prometheus rules since they have descriptions that should not be processed with the helm templates
-{{- $root := . }}
-{{- if .Values.cephClusterSpec.external }}
-{{- if .Values.cephClusterSpec.external.enable }}
-  {{- range $path, $bytes := .Files.Glob "prometheus/externalrules.yaml" }}
-  {{ $root.Files.Get $path }}
+  {{- $rulesFile := "prometheus/localrules.yaml" }}
+  {{- if and .Values.cephClusterSpec.external .Values.cephClusterSpec.external.enable }}
+    {{- $rulesFile = "prometheus/externalrules.yaml" }}
   {{- end }}
-{{- else }}
-  # The local rules are included in two different else statements because helm does not short-circuit
-  # the checks to allow the check for external settings in a single if statement.
-  {{- range $path, $bytes := .Files.Glob "prometheus/localrules.yaml" }}
-  {{ $root.Files.Get $path | indent 2 }}
+  {{- $defaultRules := .Files.Get $rulesFile | fromYaml }}
+  groups:
+  {{- range $group := $defaultRules.groups }}
+    {{- $rules := list }}
+    {{- range $rule := $group.rules }}
+      {{- $ruleName := ternary $rule.alert $rule.record (hasKey $rule "alert") }}
+      {{- $ruleOverrides := get $.Values.monitoring.prometheusRuleOverrides $ruleName }}
+      {{- if $ruleOverrides }}
+        {{- if not $ruleOverrides.disabled }}
+          {{- $ruleMerged := mergeOverwrite $rule $ruleOverrides }}
+          {{- $rules = append $rules $ruleMerged }}
+        {{- end }}
+      {{- else }}
+        {{- $rules = append $rules $rule }}
+      {{- end }}
+    {{- end }}
+    {{- if gt (len $rules) 0 }}
+    - name: {{ $group.name }}
+      rules:
+        {{- toYaml $rules | nindent 8 }}
+    {{- end }}
   {{- end }}
 {{- end }}
-{{- else }}
-  {{- range $path, $bytes := .Files.Glob "prometheus/localrules.yaml" }}
-  {{ $root.Files.Get $path | indent 2 }}
-  {{- end }}
-{{- end }}
-{{- end }}
-{{- end }}

deploy/charts/rook-ceph-cluster/values.yaml

@@ -57,6 +57,15 @@ monitoring:
   metricsDisabled: false
   # -- Whether to create the Prometheus rules for Ceph alerts
   createPrometheusRules: false
+  # -- Edit Prometheus rules for Ceph alerts
+  prometheusRuleOverrides:
+    {}
+    # CephHealthWarning:
+    #   disabled: true
+    # NVMeoFHighWriteLatency:
+    #   for: 3m
+    #   labels:
+    #     severity: critical
   # -- The namespace in which to create the prometheus rules, if different from the rook cluster namespace.
   # If you have multiple rook-ceph clusters in the same k8s cluster, choose the same namespace (ideally, namespace with prometheus
   # deployed) to set rulesNamespaceOverride for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions.