Skip to content

Commit

Permalink
Switch to DeletePartialMatch (#2660)
Browse files Browse the repository at this point in the history
Signed-off-by: Levi Harrison <git@leviharrison.dev>

Signed-off-by: Levi Harrison <git@leviharrison.dev>
  • Loading branch information
LeviHarrison committed Aug 22, 2022
1 parent ae57256 commit 9af647e
Show file tree
Hide file tree
Showing 5 changed files with 11 additions and 47 deletions.
4 changes: 1 addition & 3 deletions pkg/distributor/distributor.go
Original file line number Diff line number Diff line change
Expand Up @@ -504,9 +504,7 @@ func (d *Distributor) cleanupInactiveUser(userID string) {
d.nonHASamples.DeleteLabelValues(userID)
d.latestSeenSampleTimestampPerUser.DeleteLabelValues(userID)

if err := util.DeleteMatchingLabels(d.dedupedSamples, map[string]string{"user": userID}); err != nil {
level.Warn(d.log).Log("msg", "failed to remove cortex_distributor_deduped_samples_total metric for user", "user", userID, "err", err)
}
d.dedupedSamples.DeletePartialMatch(prometheus.Labels{"user": userID})

validation.DeletePerUserValidationMetrics(userID, d.log)
}
Expand Down
14 changes: 4 additions & 10 deletions pkg/distributor/ha_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -559,15 +559,9 @@ func findHALabels(replicaLabel, clusterLabel string, labels []mimirpb.LabelAdapt
}

func (h *haTracker) cleanupHATrackerMetricsForUser(userID string) {
filter := map[string]string{"user": userID}
filter := prometheus.Labels{"user": userID}

if err := util.DeleteMatchingLabels(h.electedReplicaChanges, filter); err != nil {
level.Warn(h.logger).Log("msg", "failed to remove cortex_ha_tracker_elected_replica_changes_total metric for user", "user", userID, "err", err)
}
if err := util.DeleteMatchingLabels(h.electedReplicaTimestamp, filter); err != nil {
level.Warn(h.logger).Log("msg", "failed to remove cortex_ha_tracker_elected_replica_timestamp_seconds metric for user", "user", userID, "err", err)
}
if err := util.DeleteMatchingLabels(h.kvCASCalls, filter); err != nil {
level.Warn(h.logger).Log("msg", "failed to remove cortex_ha_tracker_kv_store_cas_total metric for user", "user", userID, "err", err)
}
h.electedReplicaChanges.DeletePartialMatch(filter)
h.electedReplicaTimestamp.DeletePartialMatch(filter)
h.kvCASCalls.DeletePartialMatch(filter)
}
6 changes: 1 addition & 5 deletions pkg/frontend/querymiddleware/roundtrip.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"time"

"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/grafana/dskit/tenant"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
Expand Down Expand Up @@ -273,10 +272,7 @@ func newActiveUsersTripperware(logger log.Logger, registerer prometheus.Register
}, []string{"op", "user"})

activeUsers := util.NewActiveUsersCleanupWithDefaultValues(func(user string) {
err := util.DeleteMatchingLabels(queriesPerTenant, map[string]string{"user": user})
if err != nil {
level.Warn(logger).Log("msg", "failed to remove cortex_query_frontend_queries_total metric for user", "user", user)
}
queriesPerTenant.DeletePartialMatch(prometheus.Labels{"user": user})
})

// Start cleanup. If cleaner stops or fail, we will simply not clean the metrics for inactive users.
Expand Down
14 changes: 0 additions & 14 deletions pkg/util/metrics_helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -792,20 +792,6 @@ nextMetric:
return result, errs.Err()
}

// DeleteMatchingLabels removes metric with labels matching the filter.
func DeleteMatchingLabels(c CollectorVec, filter map[string]string) error {
lbls, err := GetLabels(c, filter)
if err != nil {
return err
}

for _, ls := range lbls {
c.Delete(ls.Map())
}

return nil
}

// CollectorVec is a collector that can delete metrics by labels.
// Implemented by *prometheus.MetricVec (used by CounterVec, GaugeVec, SummaryVec, and HistogramVec).
type CollectorVec interface {
Expand Down
20 changes: 5 additions & 15 deletions pkg/util/validation/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,11 @@ import (
"unicode/utf8"

"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/common/model"

"github.com/grafana/mimir/pkg/mimirpb"
"github.com/grafana/mimir/pkg/util"
"github.com/grafana/mimir/pkg/util/extract"
"github.com/grafana/mimir/pkg/util/globalerror"
)
Expand Down Expand Up @@ -275,18 +273,10 @@ func ValidateMetadata(cfg MetadataValidationConfig, userID string, metadata *mim
}

func DeletePerUserValidationMetrics(userID string, log log.Logger) {
filter := map[string]string{"user": userID}
filter := prometheus.Labels{"user": userID}

if err := util.DeleteMatchingLabels(DiscardedRequests, filter); err != nil {
level.Warn(log).Log("msg", "failed to remove cortex_discarded_requests_total metric for user", "user", userID, "err", err)
}
if err := util.DeleteMatchingLabels(DiscardedSamples, filter); err != nil {
level.Warn(log).Log("msg", "failed to remove cortex_discarded_samples_total metric for user", "user", userID, "err", err)
}
if err := util.DeleteMatchingLabels(DiscardedExemplars, filter); err != nil {
level.Warn(log).Log("msg", "failed to remove cortex_discarded_exemplars_total metric for user", "user", userID, "err", err)
}
if err := util.DeleteMatchingLabels(DiscardedMetadata, filter); err != nil {
level.Warn(log).Log("msg", "failed to remove cortex_discarded_metadata_total metric for user", "user", userID, "err", err)
}
DiscardedRequests.DeletePartialMatch(filter)
DiscardedSamples.DeletePartialMatch(filter)
DiscardedExemplars.DeletePartialMatch(filter)
DiscardedMetadata.DeletePartialMatch(filter)
}

0 comments on commit 9af647e

Please sign in to comment.