monitoring: customize prometheus rule alerts

Signed-off-by: Yuval Manor <yuvalman958@gmail.com>

Documentation/ceph-cluster-crd.md

@@ -186,6 +186,7 @@ If this value is empty, each pod will get an ephemeral directory to store their
   * `ssl`: Whether to serve the dashboard via SSL, ignored on Ceph versions older than `13.2.2`
 * `monitoring`: Settings for monitoring Ceph using Prometheus. To enable monitoring on your cluster see the [monitoring guide](ceph-monitoring.md#prometheus-alerts).
   * `enabled`: Whether to enable prometheus based monitoring for this cluster
+  * `alertRuleOverrides`: Custom prometheus rule alerts values to override default values
   * `externalMgrEndpoints`: external cluster manager endpoints
   * `externalMgrPrometheusPort`: external prometheus manager module port. See [external cluster configuration](#external-cluster) for more details.
   * `rulesNamespace`: Namespace to deploy prometheusRule. If empty, namespace of the cluster will be used.
@@ -1390,6 +1391,21 @@ spec:
     #externalMgrEndpoints:
       #- ip: 192.168.39.182
     #externalMgrPrometheusPort: 9283
+    # prometheus rule alerts values for overriding default prometheus rules values
+    # optional: specific alerts can be disabled by setting disabled field to true 
+    #alertRuleOverrides:
+    #  CephNodeDown:
+    #    disabled: true
+    #  CephMgrIsAbsent:
+    #    for: 1m
+    #    severityLevel: <custom-severityLevel> #<custom-severityLevel> must be one of the next levels: warning, critical, error
+    #    severity: custom-severity
+    #  CephOSDNearFull:
+    #    limit: 80
+    #    for: 2m
+    #  CephOSDFlapping:
+    #    osdUpRate: 10m  
+    #    severity: custom-severity-2
 ```
 
 Choose the namespace carefully, if you have an existing cluster managed by Rook, you have likely already injected `common.yaml`.

Documentation/ceph-monitoring.md

@@ -117,6 +117,21 @@ spec:
   monitoring:
     enabled: true
     rulesNamespace: "rook-ceph"
+    # prometheus rule alerts values for overriding default prometheus rules values
+    # optional: specific alerts can be disabled by setting disabled field to true 
+    #alertRuleOverrides:
+    #  CephNodeDown:
+    #    disabled: true
+    #  CephMgrIsAbsent:
+    #    for: 1m
+    #    severityLevel: <custom-severityLevel> #<custom-severityLevel> must be one of the next levels: warning, critical, error
+    #    severity: custom-severity
+    #  CephOSDNearFull:
+    #    limit: 80
+    #    for: 2m
+    #  CephOSDFlapping:
+    #    osdUpRate: 10m  
+    #    severity: custom-severity-2
 [...]
 ```
 

PendingReleaseNotes.md

@@ -4,3 +4,6 @@
 
 
 ## Features
+
+### Ceph
+- Prometheus Rule alerts can be customized by user preference.

deploy/charts/rook-ceph-cluster/values.yaml

@@ -30,6 +30,22 @@ monitoring:
   # enabling will also create RBAC rules to allow Operator to create ServiceMonitors
   enabled: false
   rulesNamespaceOverride:
+  # prometheus rule alerts values for overriding default prometheus rules values
+  # optional: specific alerts can be disabled by setting disabled field to true
+  #alertRuleOverrides:
+  #  CephNodeDown:
+  #    disabled: true
+  #  CephMgrIsAbsent:
+  #    for: 1m
+  #    severityLevel: <custom-severityLevel> #<custom-severityLevel> must be one of the next levels: warning, critical, error
+  #    severity: custom-severity
+  #  CephOSDNearFull:
+  #    limit: 80
+  #    for: 2m
+  #  CephOSDFlapping:
+  #    osdUpRate: 10m
+  #    severity: custom-severity-2
+
 
 # If true, create & use PSP resources. Set this to the same value as the rook-ceph chart.
 pspEnable: true

deploy/charts/rook-ceph/templates/resources.yaml

@@ -1593,6 +1593,32 @@ spec:
                   description: Prometheus based Monitoring settings
                   nullable: true
                   properties:
+                    alertRuleOverrides:
+                      additionalProperties:
+                        description: CephAlert basic customized alert
+                        properties:
+                          disabled:
+                            type: boolean
+                          for:
+                            type: string
+                          limit:
+                            type: integer
+                          namespace:
+                            type: string
+                          osdUpRate:
+                            type: string
+                          severity:
+                            type: string
+                          severityLevel:
+                            enum:
+                              - warning
+                              - critical
+                              - error
+                            type: string
+                        type: object
+                      description: AlertRuleOverrides points to a customized Ceph prometheus alerts
+                      nullable: true
+                      type: object
                     enabled:
                       description: Enabled determines whether to create the prometheus rules for the ceph cluster. If true, the prometheus types must exist or the creation will fail.
                       type: boolean

deploy/charts/rook-ceph/values.yaml

@@ -358,3 +358,18 @@ monitoring:
   # requires Prometheus to be pre-installed
   # enabling will also create RBAC rules to allow Operator to create ServiceMonitors
   enabled: false
+  # prometheus rule alerts values for overriding default prometheus rules values
+  # optional: specific alerts can be disabled by setting disabled field to true
+  #alertRuleOverrides:
+  #  CephNodeDown:
+  #    disabled: true
+  #  CephMgrIsAbsent:
+  #    for: 1m
+  #    severityLevel: <custom-severityLevel> #<custom-severityLevel> must be one of the next levels: warning, critical, error
+  #    severity: custom-severity
+  #  CephOSDNearFull:
+  #    limit: 80
+  #    for: 2m
+  #  CephOSDFlapping:
+  #    osdUpRate: 10m
+  #    severity: custom-severity-2

deploy/examples/cluster-external.yaml

@@ -31,3 +31,11 @@ spec:
   #   externalMgrEndpoints:
       #- ip: ip
       # externalMgrPrometheusPort: 9283
+  # prometheus rule alerts values for overriding default prometheus rules values
+  # optional: specific alerts can be disabled by setting disabled field to true
+    #alertRuleOverrides:
+    #  PersistentVolumeUsageNearFull:
+    #    limit: 80
+    #    for: 1m
+    #    severityLevel: <custom-severityLevel> #<custom-severityLevel> must be one of the next levels: warning, critical, error
+    #    severity: custom-severity

deploy/examples/cluster.yaml

@@ -79,6 +79,21 @@ spec:
     # If you have multiple rook-ceph clusters in the same k8s cluster, choose the same namespace (ideally, namespace with prometheus
     # deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions.
     rulesNamespace: rook-ceph
+    # prometheus rule alerts values for overriding default prometheus rules values
+    # optional: specific alerts can be disabled by setting disabled field to true
+    #alertRuleOverrides:
+    #  CephNodeDown:
+    #    disabled: true
+    #  CephMgrIsAbsent:
+    #    for: 1m
+    #    severityLevel: <custom-severityLevel> #<custom-severityLevel> must be one of the next levels: warning, critical, error
+    #    severity: custom-severity
+    #  CephOSDNearFull:
+    #    limit: 80
+    #    for: 2m
+    #  CephOSDFlapping:
+    #    osdUpRate: 10m
+    #    severity: custom-severity-2
   network:
     # enable host networking
     #provider: host

deploy/examples/crds.yaml

@@ -1592,6 +1592,32 @@ spec:
                   description: Prometheus based Monitoring settings
                   nullable: true
                   properties:
+                    alertRuleOverrides:
+                      additionalProperties:
+                        description: CephAlert basic customized alert
+                        properties:
+                          disabled:
+                            type: boolean
+                          for:
+                            type: string
+                          limit:
+                            type: integer
+                          namespace:
+                            type: string
+                          osdUpRate:
+                            type: string
+                          severity:
+                            type: string
+                          severityLevel:
+                            enum:
+                              - warning
+                              - critical
+                              - error
+                            type: string
+                        type: object
+                      description: AlertRuleOverrides points to a customized Ceph prometheus alerts
+                      nullable: true
+                      type: object
                     enabled:
                       description: Enabled determines whether to create the prometheus rules for the ceph cluster. If true, the prometheus types must exist or the creation will fail.
                       type: boolean