forked from ceph/ceph-mixins
-
Notifications
You must be signed in to change notification settings - Fork 0
/
monquorum.libsonnet
43 lines (43 loc) · 1.6 KB
/
monquorum.libsonnet
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
{
prometheusAlerts+:: {
groups+: [
{
name: 'quorum-alert.rules',
rules: [
{
alert: 'CephMonQuorumAtRisk',
expr: |||
count(ceph_mon_quorum_status{%s} == 1) by (namespace) <= (floor(count(ceph_mon_metadata{%s}) by (namespace) / 2) + 1)
||| % [$._config.cephExporterSelector, $._config.cephExporterSelector],
'for': $._config.monQuorumAlertTime,
labels: {
severity: 'critical',
},
annotations: {
message: 'Storage quorum at risk',
description: 'Storage cluster quorum is low. Contact Support.',
storage_type: $._config.storageType,
severity_level: 'error',
},
},
{
alert: 'CephMonHighNumberOfLeaderChanges',
expr: |||
(ceph_mon_metadata{%(cephExporterSelector)s} * on (ceph_daemon) group_left() (rate(ceph_mon_num_elections{%(cephExporterSelector)s}[5m]) * 60)) > 0.95
||| % $._config,
'for': $._config.monQuorumLeaderChangesAlertTime,
labels: {
severity: 'warning',
},
annotations: {
message: 'Storage Cluster has seen many leader changes recently.',
description: 'Ceph Monitor {{ $labels.ceph_daemon }} on host {{ $labels.hostname }} has seen {{ $value | printf "%.2f" }} leader changes per minute recently.',
storage_type: $._config.storageType,
severity_level: 'warning',
},
},
],
},
],
},
}