-
Notifications
You must be signed in to change notification settings - Fork 74
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[alerting] Add severity and refactor code (#569)
* Move AlertInfo to its own package so that it can shared across notifier implementations. * Remove "json" from alert fields. It's not useful for most of the notifier types. * Add "severity" to alert configuration. This will be included in the alert details and will be used for PagerDuty events. * Add "other_info" to include additional information about alerts. This information can include fields like: team, additional dashboards, etc. * Include all fields as details in the Slack and Email messages. PagerDuty already included all fields in custom details field.
- Loading branch information
Showing
18 changed files
with
592 additions
and
216 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| // Copyright 2023 The Cloudprober Authors. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // Package alertinfo implements AlertInfo struct for sharing alert data | ||
| // across modules. | ||
| package alertinfo | ||
|
|
||
| import ( | ||
| "sort" | ||
| "strconv" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/cloudprober/cloudprober/common/strtemplate" | ||
| "github.com/cloudprober/cloudprober/targets/endpoint" | ||
| ) | ||
|
|
||
| // AlertInfo contains information about an alert. | ||
| type AlertInfo struct { | ||
| Name string | ||
| ProbeName string | ||
| ConditionID string | ||
| Target endpoint.Endpoint | ||
| Failures int | ||
| Total int | ||
| FailingSince time.Time | ||
| } | ||
|
|
||
| func (ai *AlertInfo) Fields(templateDetails map[string]string) map[string]string { | ||
| fields := map[string]string{ | ||
| "alert": ai.Name, | ||
| "probe": ai.ProbeName, | ||
| "target": ai.Target.Dst(), | ||
| "condition_id": ai.ConditionID, | ||
| "failures": strconv.Itoa(ai.Failures), | ||
| "total": strconv.Itoa(ai.Total), | ||
| "since": ai.FailingSince.Format(time.RFC3339), | ||
| } | ||
|
|
||
| for k, v := range ai.Target.Labels { | ||
| fields["target.label."+k] = v | ||
| } | ||
|
|
||
| if ai.Target.IP != nil { | ||
| fields["target_ip"] = ai.Target.IP.String() | ||
| } | ||
|
|
||
| // Note that we parse details in the end, that's because details template | ||
| // may use other parsed fields like dashboard_url, playbook_url, etc. | ||
| for k, v := range templateDetails { | ||
| if k != "details" { | ||
| fields[k], _ = strtemplate.SubstituteLabels(v, fields) | ||
| } | ||
| } | ||
| if templateDetails["details"] != "" { | ||
| fields["details"], _ = strtemplate.SubstituteLabels(templateDetails["details"], fields) | ||
| } | ||
|
|
||
| return fields | ||
| } | ||
|
|
||
| func FieldsToString(fields map[string]string, skipKeys ...string) string { | ||
| skipMap := make(map[string]bool) | ||
| for _, k := range skipKeys { | ||
| skipMap[k] = true | ||
| } | ||
|
|
||
| var keys []string | ||
| for k := range fields { | ||
| if !skipMap[k] { | ||
| keys = append(keys, k) | ||
| } | ||
| } | ||
|
|
||
| sort.Strings(keys) | ||
|
|
||
| var out []string | ||
| for _, k := range keys { | ||
| out = append(out, k+": "+fields[k]) | ||
| } | ||
|
|
||
| return strings.Join(out, "\n") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| // Copyright 2023 The Cloudprober Authors. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // Package alertinfo implements AlertInfo struct for sharing alert data | ||
| // across modules. | ||
| package alertinfo | ||
|
|
||
| import ( | ||
| "net" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/cloudprober/cloudprober/targets/endpoint" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestAlertInfoFields(t *testing.T) { | ||
| testTarget := endpoint.Endpoint{ | ||
| Name: "test-target", | ||
| IP: net.ParseIP("10.11.12.13"), | ||
| Labels: map[string]string{ | ||
| "apptype": "backend", | ||
| "language": "go", | ||
| }, | ||
| } | ||
|
|
||
| tests := []struct { | ||
| name string | ||
| ai *AlertInfo | ||
| templateDetails map[string]string | ||
| want map[string]string | ||
| }{ | ||
| { | ||
| name: "no_template_details", | ||
| ai: &AlertInfo{ | ||
| Name: "test-alert", | ||
| ProbeName: "test-probe", | ||
| ConditionID: "122333444", | ||
| Target: testTarget, | ||
| Failures: 8, | ||
| Total: 12, | ||
| FailingSince: time.Time{}.Add(time.Second), | ||
| }, | ||
| want: map[string]string{ | ||
| "alert": "test-alert", | ||
| "probe": "test-probe", | ||
| "condition_id": "122333444", | ||
| "target": "test-target", | ||
| "target_ip": "10.11.12.13", | ||
| "failures": "8", | ||
| "total": "12", | ||
| "since": "0001-01-01T00:00:01Z", | ||
| "target.label.apptype": "backend", | ||
| "target.label.language": "go", | ||
| }, | ||
| }, | ||
| { | ||
| name: "with_template_details", | ||
| ai: &AlertInfo{ | ||
| Name: "test-alert", | ||
| ProbeName: "test-probe", | ||
| ConditionID: "122333444", | ||
| Target: testTarget, | ||
| Failures: 8, | ||
| Total: 12, | ||
| FailingSince: time.Time{}.Add(time.Second), | ||
| }, | ||
| templateDetails: map[string]string{ | ||
| "summary": "Cloudprober alert \"@alert@\" for \"@target@\"", | ||
| "dashboard_url": "https://my-dashboard.com/probe=@probe@&target=@target@", | ||
| "details": "Dashboard: @dashboard_url@", | ||
| }, | ||
| want: map[string]string{ | ||
| "alert": "test-alert", | ||
| "probe": "test-probe", | ||
| "condition_id": "122333444", | ||
| "target": "test-target", | ||
| "target_ip": "10.11.12.13", | ||
| "failures": "8", | ||
| "total": "12", | ||
| "since": "0001-01-01T00:00:01Z", | ||
| "target.label.apptype": "backend", | ||
| "target.label.language": "go", | ||
| "summary": "Cloudprober alert \"test-alert\" for \"test-target\"", | ||
| "dashboard_url": "https://my-dashboard.com/probe=test-probe&target=test-target", | ||
| "details": "Dashboard: https://my-dashboard.com/probe=test-probe&target=test-target", | ||
| }, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| assert.Equal(t, tt.want, tt.ai.Fields(tt.templateDetails), "Fields don't match") | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestFieldsToString(t *testing.T) { | ||
| fields := map[string]string{ | ||
| "alert": "test-alert", | ||
| "probe": "test-probe", | ||
| "condition_id": "122333444", | ||
| } | ||
|
|
||
| tests := []struct { | ||
| name string | ||
| skipKeys []string | ||
| want string | ||
| }{ | ||
| { | ||
| name: "skip_none", | ||
| want: "alert: test-alert\ncondition_id: 122333444\nprobe: test-probe", | ||
| }, | ||
| { | ||
| name: "skip_probe", | ||
| skipKeys: []string{"condition_id"}, | ||
| want: "alert: test-alert\nprobe: test-probe", | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| assert.Equal(t, tt.want, FieldsToString(fields, tt.skipKeys...), "Fields don't match") | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.