Merge pull request #2042 from eyenx/feat/add-prometheus-rules-for-pro…

…mtail

[promtail] Add PrometheusRules functionality

charts/promtail/Chart.yaml

@@ -3,7 +3,7 @@ name: promtail
 description: Promtail is an agent which ships the contents of local logs to a Loki instance
 type: application
 appVersion: 2.7.0
-version: 6.7.1
+version: 6.7.2
 home: https://grafana.com/loki
 sources:
   - https://github.com/grafana/loki

charts/promtail/README.md

@@ -1,6 +1,6 @@
 # promtail
 
-![Version: 6.7.1](https://img.shields.io/badge/Version-6.7.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.7.0](https://img.shields.io/badge/AppVersion-2.7.0-informational?style=flat-square)
+![Version: 6.7.2](https://img.shields.io/badge/Version-6.7.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.7.0](https://img.shields.io/badge/AppVersion-2.7.0-informational?style=flat-square)
 
 Promtail is an agent which ships the contents of local logs to a Loki instance
 
@@ -140,6 +140,7 @@ The new release which will pick up again from the existing `positions.yaml`.
 | serviceMonitor.metricRelabelings | list | `[]` | ServiceMonitor relabel configs to apply to samples as the last step before ingestion https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig (defines `metric_relabel_configs`) |
 | serviceMonitor.namespace | string | `nil` | Alternative namespace for ServiceMonitor resources |
 | serviceMonitor.namespaceSelector | object | `{}` | Namespace selector for ServiceMonitor resources |
+| serviceMonitor.prometheusRule | object | `{"additionalLabels":{},"enabled":false,"rules":[]}` | Prometheus rules will be deployed for alerting purposes |
 | serviceMonitor.relabelings | list | `[]` | ServiceMonitor relabel configs to apply to samples before scraping https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig (defines `relabel_configs`) |
 | serviceMonitor.scheme | string | `"http"` | ServiceMonitor will use http by default, but you can pick https as well |
 | serviceMonitor.scrapeTimeout | string | `nil` | ServiceMonitor scrape timeout in Go duration format (e.g. 15s) |

charts/promtail/templates/prometheus-rules.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.serviceMonitor.enabled .Values.serviceMonitor.prometheusRule.enabled -}}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "promtail.fullname" . }}
+  {{- with .Values.serviceMonitor.prometheusRule.namespace }}
+  namespace: {{ . | quote }}
+  {{- end }}
+  labels:
+    {{- include "promtail.labels" . | nindent 4 }}
+    {{- with .Values.serviceMonitor.prometheusRule.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+{{- if .Values.serviceMonitor.prometheusRule.rules }}
+  groups:
+  - name: {{ template "promtail.fullname" . }}
+    rules:
+    {{- toYaml .Values.serviceMonitor.prometheusRule.rules | nindent 4 }}
+{{- end }}
+{{- end }}

charts/promtail/values.yaml

@@ -223,6 +223,36 @@ serviceMonitor:
   scheme: http
   # -- ServiceMonitor will use these tlsConfig settings to make the health check requests
   tlsConfig: null
+  # -- Prometheus rules will be deployed for alerting purposes
+  prometheusRule:
+    enabled: false
+    additionalLabels: {}
+    # namespace:
+    rules: []
+    #  - alert: PromtailRequestErrors
+    #    expr: 100 * sum(rate(promtail_request_duration_seconds_count{status_code=~"5..|failed"}[1m])) by (namespace, job, route, instance) / sum(rate(promtail_request_duration_seconds_count[1m])) by (namespace, job, route, instance) > 10
+    #    for: 5m
+    #    labels:
+    #      severity: critical
+    #    annotations:
+    #      description: |
+    #        The {{ $labels.job }} {{ $labels.route }} is experiencing
+    #        {{ printf \"%.2f\" $value }} errors.
+    #        VALUE = {{ $value }}
+    #        LABELS = {{ $labels }}
+    #      summary: Promtail request errors (instance {{ $labels.instance }})
+    #  - alert: PromtailRequestLatency
+    #    expr: histogram_quantile(0.99, sum(rate(promtail_request_duration_seconds_bucket[5m])) by (le)) > 1
+    #    for: 5m
+    #    labels:
+    #      severity: critical
+    #    annotations:
+    #      summary: Promtail request latency (instance {{ $labels.instance }})
+    #      description: |
+    #        The {{ $labels.job }} {{ $labels.route }} is experiencing
+    #        {{ printf \"%.2f\" $value }}s 99th percentile latency.
+    #        VALUE = {{ $value }}
+    #        LABELS = {{ $labels }}
 
 # Extra containers created as part of a Promtail Deployment resource
 # - spec for Container: