From a2f9e125513ac9cf198a5042503f89f6c5a11d05 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Fri, 14 Oct 2022 08:24:31 -0700 Subject: [PATCH] [receiver/statsd] Add an exponential histogram option (#12951) Adds a "histogram" option to enable the OTLP v0.11 auto-scaling exponential histogram aggregator for Timing and Histogram measurements. Co-authored-by: Pablo Baeyens --- .chloggen/expo_statsd_receiver.yaml | 4 + cmd/configschema/go.mod | 1 + cmd/configschema/go.sum | 2 + go.mod | 1 + go.sum | 2 + .../coreinternal/goldendataset/metrics_gen.go | 9 +- .../metricstestutil/metric_diff.go | 6 + .../metricstestutil/metric_diff_test.go | 6 +- receiver/statsdreceiver/README.md | 16 +- receiver/statsdreceiver/config.go | 16 +- receiver/statsdreceiver/config_test.go | 6 +- receiver/statsdreceiver/factory_test.go | 92 +++++++ receiver/statsdreceiver/go.mod | 10 +- receiver/statsdreceiver/go.sum | 26 +- .../protocol/metric_translator.go | 50 +++- .../statsdreceiver/protocol/statsd_parser.go | 118 +++++++-- .../protocol/statsd_parser_test.go | 228 +++++++++++++++++- receiver/statsdreceiver/testdata/config.yaml | 4 +- 18 files changed, 528 insertions(+), 69 deletions(-) create mode 100755 .chloggen/expo_statsd_receiver.yaml diff --git a/.chloggen/expo_statsd_receiver.yaml b/.chloggen/expo_statsd_receiver.yaml new file mode 100755 index 0000000000000..c4074a5b2a158 --- /dev/null +++ b/.chloggen/expo_statsd_receiver.yaml @@ -0,0 +1,4 @@ +change_type: enhancement +component: receiver/statsdreceiver +note: "Add OTLP exponential histogram aggregator support for high-resolution histogram and timing metrics" +issues: [5742] diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod index feb51c8c2afef..5a6a484df0e95 100644 --- a/cmd/configschema/go.mod +++ b/cmd/configschema/go.mod @@ -271,6 +271,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/leoluk/perflib_exporter v0.2.0 // indirect github.com/lib/pq v1.10.7 // indirect + github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/linkedin/goavro/v2 v2.9.8 // indirect github.com/linode/linodego v1.8.0 // indirect github.com/lufia/plan9stats v0.0.0-20220517141722-cf486979b281 // indirect diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum index 5b9f156e54ffa..15c7f0bc21ffa 100644 --- a/cmd/configschema/go.sum +++ b/cmd/configschema/go.sum @@ -1346,6 +1346,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lightstep/go-expohisto v1.0.0 h1:UPtTS1rGdtehbbAF7o/dhkWLTDI73UifG8LbfQI7cA4= +github.com/lightstep/go-expohisto v1.0.0/go.mod h1:xDXD0++Mu2FOaItXtdDfksfgxfV0z1TMPa+e/EUd0cs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= diff --git a/go.mod b/go.mod index d9ad0b207c84e..d3d703a8d3d64 100644 --- a/go.mod +++ b/go.mod @@ -422,6 +422,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/leoluk/perflib_exporter v0.2.0 // indirect github.com/lib/pq v1.10.7 // indirect + github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/linkedin/goavro/v2 v2.9.8 // indirect github.com/linode/linodego v1.8.0 // indirect github.com/lufia/plan9stats v0.0.0-20220517141722-cf486979b281 // indirect diff --git a/go.sum b/go.sum index fc241fd523fee..ac8406feec07f 100644 --- a/go.sum +++ b/go.sum @@ -1344,6 +1344,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lightstep/go-expohisto v1.0.0 h1:UPtTS1rGdtehbbAF7o/dhkWLTDI73UifG8LbfQI7cA4= +github.com/lightstep/go-expohisto v1.0.0/go.mod h1:xDXD0++Mu2FOaItXtdDfksfgxfV0z1TMPa+e/EUd0cs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= diff --git a/internal/coreinternal/goldendataset/metrics_gen.go b/internal/coreinternal/goldendataset/metrics_gen.go index 9bc4087b90320..3bd233fe56e47 100644 --- a/internal/coreinternal/goldendataset/metrics_gen.go +++ b/internal/coreinternal/goldendataset/metrics_gen.go @@ -192,6 +192,7 @@ func setDoubleHistogramBounds(hdp pmetric.HistogramDataPoint, bounds ...float64) func addDoubleHistogramVal(hdp pmetric.HistogramDataPoint, val float64) { hdp.SetCount(hdp.Count() + 1) hdp.SetSum(hdp.Sum() + val) + // TODO: HasSum, Min, HasMin, Max, HasMax are not covered in tests. buckets := hdp.BucketCounts() bounds := hdp.ExplicitBounds() for i := 0; i < bounds.Len(); i++ { @@ -225,10 +226,12 @@ func populateExpoHistogram(cfg MetricsCfg, dh pmetric.ExponentialHistogram) { pt.SetTimestamp(ts) populatePtAttributes(cfg, pt.Attributes()) - pt.SetSum(100) + pt.SetSum(100 * float64(cfg.PtVal)) pt.SetCount(uint64(cfg.PtVal)) - pt.SetScale(0) - pt.SetZeroCount(0) + pt.SetScale(int32(cfg.PtVal)) + pt.SetZeroCount(uint64(cfg.PtVal)) + pt.SetMin(float64(cfg.PtVal)) + pt.SetMax(float64(cfg.PtVal)) pt.Positive().SetOffset(int32(cfg.PtVal)) pt.Positive().BucketCounts().FromRaw([]uint64{uint64(cfg.PtVal)}) } diff --git a/internal/coreinternal/metricstestutil/metric_diff.go b/internal/coreinternal/metricstestutil/metric_diff.go index 9966cdf3e63fe..0911099f11094 100644 --- a/internal/coreinternal/metricstestutil/metric_diff.go +++ b/internal/coreinternal/metricstestutil/metric_diff.go @@ -206,6 +206,7 @@ func diffHistogramPt( diffs = diffMetricAttrs(diffs, expected.Attributes(), actual.Attributes()) diffs = diff(diffs, expected.Count(), actual.Count(), "HistogramDataPoint Count") diffs = diff(diffs, expected.Sum(), actual.Sum(), "HistogramDataPoint Sum") + // TODO: HasSum, Min, HasMin, Max, HasMax are not covered in tests. diffs = diff(diffs, expected.BucketCounts(), actual.BucketCounts(), "HistogramDataPoint BucketCounts") diffs = diff(diffs, expected.ExplicitBounds(), actual.ExplicitBounds(), "HistogramDataPoint ExplicitBounds") return diffExemplars(diffs, expected.Exemplars(), actual.Exemplars()) @@ -234,7 +235,12 @@ func diffExponentialHistogramPt( ) []*MetricDiff { diffs = diffMetricAttrs(diffs, expected.Attributes(), actual.Attributes()) diffs = diff(diffs, expected.Count(), actual.Count(), "ExponentialHistogramDataPoint Count") + diffs = diff(diffs, expected.HasSum(), actual.HasSum(), "ExponentialHistogramDataPoint HasSum") + diffs = diff(diffs, expected.HasMin(), actual.HasMin(), "ExponentialHistogramDataPoint HasMin") + diffs = diff(diffs, expected.HasMax(), actual.HasMax(), "ExponentialHistogramDataPoint HasMax") diffs = diff(diffs, expected.Sum(), actual.Sum(), "ExponentialHistogramDataPoint Sum") + diffs = diff(diffs, expected.Min(), actual.Min(), "ExponentialHistogramDataPoint Min") + diffs = diff(diffs, expected.Max(), actual.Max(), "ExponentialHistogramDataPoint Max") diffs = diff(diffs, expected.ZeroCount(), actual.ZeroCount(), "ExponentialHistogramDataPoint ZeroCount") diffs = diff(diffs, expected.Scale(), actual.Scale(), "ExponentialHistogramDataPoint Scale") diff --git a/internal/coreinternal/metricstestutil/metric_diff_test.go b/internal/coreinternal/metricstestutil/metric_diff_test.go index ef2896b6090b4..1aae2b7db8293 100644 --- a/internal/coreinternal/metricstestutil/metric_diff_test.go +++ b/internal/coreinternal/metricstestutil/metric_diff_test.go @@ -84,13 +84,13 @@ func TestAttributes(t *testing.T) { func TestExponentialHistogram(t *testing.T) { cfg1 := goldendataset.DefaultCfg() - cfg1.MetricDescriptorType = pmetric.MetricTypeHistogram + cfg1.MetricDescriptorType = pmetric.MetricTypeExponentialHistogram cfg1.PtVal = 1 expected := goldendataset.MetricsFromCfg(cfg1) cfg2 := goldendataset.DefaultCfg() - cfg2.MetricDescriptorType = pmetric.MetricTypeHistogram + cfg2.MetricDescriptorType = pmetric.MetricTypeExponentialHistogram cfg2.PtVal = 3 actual := goldendataset.MetricsFromCfg(cfg2) diffs := DiffMetrics(nil, expected, actual) - assert.Len(t, diffs, 3) + assert.Len(t, diffs, 8) } diff --git a/receiver/statsdreceiver/README.md b/receiver/statsdreceiver/README.md index d5a19e3035297..1bff380205e2a 100644 --- a/receiver/statsdreceiver/README.md +++ b/receiver/statsdreceiver/README.md @@ -30,9 +30,9 @@ The Following settings are optional: `"statsd_type"` specifies received Statsd data type. Possible values for this setting are `"timing"`, `"timer"` and `"histogram"`. -`"observer_type"` specifies OTLP data type to convert to. We support `"gauge"` and `"summary"`. For `"gauge"`, it does not perform any aggregation. -For `"summary`, the statsD receiver will aggregate to one OTLP summary metric for one metric description(the same metric name with the same tags). It will send percentile 0, 10, 50, 90, 95, 100 to the downstream. -TODO: Add a new option to use a smoothed summary like Promethetheus: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/3261 +`"observer_type"` specifies OTLP data type to convert to. We support `"gauge"`, `"summary"`, and `"histogram"`. For `"gauge"`, it does not perform any aggregation. +For `"summary`, the statsD receiver will aggregate to one OTLP summary metric for one metric description (the same metric name with the same tags). It will send percentile 0, 10, 50, 90, 95, 100 to the downstream. The `"histogram"` setting selects an [auto-scaling exponential histogram configured with only a maximum size](https://github.com/lightstep/go-expohisto#readme), as shown in the example below. +TODO: Add a new option to use a smoothed summary like Prometheus: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/3261 Example: @@ -48,7 +48,9 @@ receivers: - statsd_type: "histogram" observer_type: "gauge" - statsd_type: "timing" - observer_type: "gauge" + observer_type: "histogram" + histogram: + max_size: 100 ``` The full list of settings exposed for this receiver are documented [here](./config.go) @@ -123,9 +125,11 @@ receivers: is_monotonic_counter: false # default timer_histogram_mapping: - statsd_type: "histogram" - observer_type: "gauge" + observer_type: "histogram" + histogram: + max_size: 50 - statsd_type: "timing" - observer_type: "gauge" + observer_type: "summary" exporters: file: diff --git a/receiver/statsdreceiver/config.go b/receiver/statsdreceiver/config.go index 62f62ede94aed..5bf81c6f1b558 100644 --- a/receiver/statsdreceiver/config.go +++ b/receiver/statsdreceiver/config.go @@ -18,6 +18,7 @@ import ( "fmt" "time" + "github.com/lightstep/go-expohisto/structure" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/config/confignet" "go.uber.org/multierr" @@ -36,7 +37,6 @@ type Config struct { } func (c *Config) validate() error { - var errs error if c.AggregationInterval <= 0 { @@ -63,10 +63,22 @@ func (c *Config) validate() error { } switch eachMap.ObserverType { - case protocol.GaugeObserver, protocol.SummaryObserver: + case protocol.GaugeObserver, protocol.SummaryObserver, protocol.HistogramObserver: default: errs = multierr.Append(errs, fmt.Errorf("observer_type is not supported: %s", eachMap.ObserverType)) } + + if eachMap.ObserverType == protocol.HistogramObserver { + if eachMap.Histogram.MaxSize != 0 && (eachMap.Histogram.MaxSize < structure.MinSize || eachMap.Histogram.MaxSize > structure.MaximumMaxSize) { + errs = multierr.Append(errs, fmt.Errorf("histogram max_size out of range: %v", eachMap.Histogram.MaxSize)) + } + } else { + // Non-histogram observer w/ histogram config + var empty protocol.HistogramConfig + if eachMap.Histogram != empty { + errs = multierr.Append(errs, fmt.Errorf("histogram configuration requires observer_type: histogram")) + } + } } if TimerHistogramMappingMissingObjectName { diff --git a/receiver/statsdreceiver/config_test.go b/receiver/statsdreceiver/config_test.go index d2042d1c8df74..ec85260623956 100644 --- a/receiver/statsdreceiver/config_test.go +++ b/receiver/statsdreceiver/config_test.go @@ -59,7 +59,10 @@ func TestLoadConfig(t *testing.T) { }, { StatsdType: "timing", - ObserverType: "gauge", + ObserverType: "histogram", + Histogram: protocol.HistogramConfig{ + MaxSize: 170, + }, }, }, }, @@ -153,5 +156,4 @@ func TestValidate(t *testing.T) { require.EqualError(t, test.cfg.validate(), test.expectedErr) }) } - } diff --git a/receiver/statsdreceiver/factory_test.go b/receiver/statsdreceiver/factory_test.go index bb6389e9a1090..24a6122304a66 100644 --- a/receiver/statsdreceiver/factory_test.go +++ b/receiver/statsdreceiver/factory_test.go @@ -17,8 +17,11 @@ package statsdreceiver import ( "context" "testing" + "time" + "github.com/lightstep/go-expohisto/structure" "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config/configtest" "go.opentelemetry.io/collector/consumer/consumertest" @@ -26,6 +29,18 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver/protocol" ) +type testHost struct { + component.Host + t *testing.T +} + +// ReportFatalError causes the test to be run to fail. +func (h *testHost) ReportFatalError(err error) { + h.t.Fatalf("receiver reported a fatal error: %v", err) +} + +var _ component.Host = (*testHost)(nil) + func TestCreateDefaultConfig(t *testing.T) { factory := NewFactory() cfg := factory.CreateDefaultConfig() @@ -61,6 +76,83 @@ func TestCreateReceiverWithConfigErr(t *testing.T) { } +func TestCreateReceiverWithHistogramConfigError(t *testing.T) { + for _, maxSize := range []int32{structure.MaximumMaxSize + 1, -1, -structure.MaximumMaxSize} { + cfg := &Config{ + AggregationInterval: 20 * time.Second, + TimerHistogramMapping: []protocol.TimerHistogramMapping{ + { + StatsdType: "timing", + ObserverType: "histogram", + Histogram: protocol.HistogramConfig{ + MaxSize: maxSize, + }, + }, + }, + } + receiver, err := createMetricsReceiver( + context.Background(), + componenttest.NewNopReceiverCreateSettings(), + cfg, + consumertest.NewNop(), + ) + assert.Error(t, err) + assert.Contains(t, err.Error(), "histogram max_size out of range") + assert.Nil(t, receiver) + } +} + +func TestCreateReceiverWithHistogramGoodConfig(t *testing.T) { + for _, maxSize := range []int32{structure.MaximumMaxSize, 0, 2} { + cfg := &Config{ + AggregationInterval: 20 * time.Second, + TimerHistogramMapping: []protocol.TimerHistogramMapping{ + { + StatsdType: "timing", + ObserverType: "histogram", + Histogram: protocol.HistogramConfig{ + MaxSize: maxSize, + }, + }, + }, + } + receiver, err := createMetricsReceiver( + context.Background(), + componenttest.NewNopReceiverCreateSettings(), + cfg, + consumertest.NewNop(), + ) + assert.NoError(t, err) + assert.NotNil(t, receiver) + assert.NoError(t, receiver.Start(context.Background(), &testHost{t: t})) + assert.NoError(t, receiver.Shutdown(context.Background())) + } +} + +func TestCreateReceiverWithInvalidHistogramConfig(t *testing.T) { + cfg := &Config{ + AggregationInterval: 20 * time.Second, + TimerHistogramMapping: []protocol.TimerHistogramMapping{ + { + StatsdType: "timing", + ObserverType: "gauge", + Histogram: protocol.HistogramConfig{ + MaxSize: 100, + }, + }, + }, + } + receiver, err := createMetricsReceiver( + context.Background(), + componenttest.NewNopReceiverCreateSettings(), + cfg, + consumertest.NewNop(), + ) + assert.Error(t, err) + assert.Contains(t, err.Error(), "histogram configuration requires observer_type: histogram") + assert.Nil(t, receiver) +} + func TestCreateMetricsReceiverWithNilConsumer(t *testing.T) { receiver, err := createMetricsReceiver( context.Background(), diff --git a/receiver/statsdreceiver/go.mod b/receiver/statsdreceiver/go.mod index 331cba6dd9e7c..38deb14d01afd 100644 --- a/receiver/statsdreceiver/go.mod +++ b/receiver/statsdreceiver/go.mod @@ -3,7 +3,9 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsd go 1.18 require ( + github.com/lightstep/go-expohisto v1.0.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.62.0 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.62.0 github.com/stretchr/testify v1.8.0 go.opencensus.io v0.23.0 go.opentelemetry.io/collector v0.62.0 @@ -16,7 +18,6 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -30,20 +31,21 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/otel/metric v0.32.1 // indirect go.opentelemetry.io/otel/sdk v1.10.0 // indirect go.opentelemetry.io/otel/trace v1.10.0 // indirect go.uber.org/atomic v1.10.0 // indirect golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 // indirect golang.org/x/text v0.3.7 // indirect - google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect + google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect google.golang.org/grpc v1.50.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/common => ../../internal/common + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => ../../internal/coreinternal diff --git a/receiver/statsdreceiver/go.sum b/receiver/statsdreceiver/go.sum index b1353f44b87dd..31bbe267b24dc 100644 --- a/receiver/statsdreceiver/go.sum +++ b/receiver/statsdreceiver/go.sum @@ -3,7 +3,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -30,12 +29,10 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -46,16 +43,13 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -108,7 +102,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -176,6 +170,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lightstep/go-expohisto v1.0.0 h1:UPtTS1rGdtehbbAF7o/dhkWLTDI73UifG8LbfQI7cA4= +github.com/lightstep/go-expohisto v1.0.0/go.mod h1:xDXD0++Mu2FOaItXtdDfksfgxfV0z1TMPa+e/EUd0cs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -218,7 +214,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -254,7 +249,6 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -281,6 +275,7 @@ go.opentelemetry.io/collector v0.62.0 h1:SdCUsT69mHhEkYKPTv3urzMemwF8pbX7zqSYP5H go.opentelemetry.io/collector v0.62.0/go.mod h1:Qs172nQ5pfK13EvDPNpYPLB2nvp4sR4MfF9eciUMwWE= go.opentelemetry.io/collector/pdata v0.62.0 h1:7M2512nLih9UXR+DvWo84UQFES9M7Hh5lR3odxhAGUY= go.opentelemetry.io/collector/pdata v0.62.0/go.mod h1:ziGuxiR4TVSZ7pT+j1t58zYFVQtWwiWi9ng9EFmp5U0= +go.opentelemetry.io/collector/semconv v0.62.0 h1:Zc5Nt+kxVZKftwkOFo9VUAVPILCtLasvdkqV2fJIH0Y= go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= go.opentelemetry.io/otel/metric v0.32.1 h1:ftff5LSBCIDwL0UkhBuDg8j9NNxx2IusvJ18q9h6RC4= @@ -289,7 +284,6 @@ go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpT go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -339,8 +333,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -385,7 +379,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs= golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -424,8 +417,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc h1:Nf+EdcTLHR8qDNN/KfkQL0u0ssxt9OhbaWCl5C0ucEI= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -434,9 +427,7 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.50.0 h1:fPVVDxY9w++VjTZsYvXWqEf9Rqar/e+9zYfxKK+W+YU= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -450,7 +441,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/receiver/statsdreceiver/protocol/metric_translator.go b/receiver/statsdreceiver/protocol/metric_translator.go index 76084eb6acf17..d96394c8d74d6 100644 --- a/receiver/statsdreceiver/protocol/metric_translator.go +++ b/receiver/statsdreceiver/protocol/metric_translator.go @@ -18,6 +18,7 @@ import ( "sort" "time" + "github.com/lightstep/go-expohisto/structure" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "gonum.org/v1/gonum/stat" @@ -98,6 +99,51 @@ func buildSummaryMetric(desc statsDMetricDescription, summary summaryMetric, sta } } +func buildHistogramMetric(desc statsDMetricDescription, histogram histogramMetric, startTime, timeNow time.Time, ilm pmetric.ScopeMetrics) { + nm := ilm.Metrics().AppendEmpty() + nm.SetName(desc.name) + expo := nm.SetEmptyExponentialHistogram() + expo.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + + dp := expo.DataPoints().AppendEmpty() + agg := histogram.agg + + dp.SetCount(agg.Count()) + dp.SetSum(agg.Sum()) + if agg.Count() != 0 { + dp.SetMin(agg.Min()) + dp.SetMax(agg.Max()) + } + + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(startTime)) + dp.SetTimestamp(pcommon.NewTimestampFromTime(timeNow)) + + for i := desc.attrs.Iter(); i.Next(); { + dp.Attributes().PutStr(string(i.Attribute().Key), i.Attribute().Value.AsString()) + } + + dp.SetZeroCount(agg.ZeroCount()) + dp.SetScale(agg.Scale()) + + for _, half := range []struct { + inFunc func() *structure.Buckets + outFunc func() pmetric.ExponentialHistogramDataPointBuckets + }{ + {agg.Positive, dp.Positive}, + {agg.Negative, dp.Negative}, + } { + in := half.inFunc() + out := half.outFunc() + out.SetOffset(in.Offset()) + + out.BucketCounts().EnsureCapacity(int(in.Len())) + + for i := uint32(0); i < in.Len(); i++ { + out.BucketCounts().Append(in.At(i)) + } + } +} + func (s statsDMetric) counterValue() int64 { x := s.asFloat // Note statds counters are always represented as integers. @@ -116,12 +162,12 @@ func (s statsDMetric) gaugeValue() float64 { return s.asFloat } -func (s statsDMetric) summaryValue() summaryRaw { +func (s statsDMetric) sampleValue() sampleValue { count := 1.0 if 0 < s.sampleRate && s.sampleRate < 1 { count /= s.sampleRate } - return summaryRaw{ + return sampleValue{ value: s.asFloat, count: count, } diff --git a/receiver/statsdreceiver/protocol/statsd_parser.go b/receiver/statsdreceiver/protocol/statsd_parser.go index 4577f4b004008..a7e31523509e9 100644 --- a/receiver/statsdreceiver/protocol/statsd_parser.go +++ b/receiver/statsdreceiver/protocol/statsd_parser.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/lightstep/go-expohisto/structure" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/otel/attribute" ) @@ -50,16 +51,31 @@ const ( TimingTypeName TypeName = "timing" TimingAltTypeName TypeName = "timer" - GaugeObserver ObserverType = "gauge" - SummaryObserver ObserverType = "summary" - DisableObserver ObserverType = "disabled" + GaugeObserver ObserverType = "gauge" + SummaryObserver ObserverType = "summary" + HistogramObserver ObserverType = "histogram" + DisableObserver ObserverType = "disabled" DefaultObserverType = DisableObserver ) type TimerHistogramMapping struct { - StatsdType TypeName `mapstructure:"statsd_type"` - ObserverType ObserverType `mapstructure:"observer_type"` + StatsdType TypeName `mapstructure:"statsd_type"` + ObserverType ObserverType `mapstructure:"observer_type"` + Histogram HistogramConfig `mapstructure:"histogram"` +} + +type HistogramConfig struct { + MaxSize int32 `mapstructure:"max_size"` +} + +type ObserverCategory struct { + method ObserverType + histogramConfig structure.Config +} + +var defaultObserverCategory = ObserverCategory{ + method: DefaultObserverType, } // StatsDParser supports the Parse method for parsing StatsD messages with Tags. @@ -67,15 +83,16 @@ type StatsDParser struct { gauges map[statsDMetricDescription]pmetric.ScopeMetrics counters map[statsDMetricDescription]pmetric.ScopeMetrics summaries map[statsDMetricDescription]summaryMetric + histograms map[statsDMetricDescription]histogramMetric timersAndDistributions []pmetric.ScopeMetrics enableMetricType bool isMonotonicCounter bool - observeTimer ObserverType - observeHistogram ObserverType + timerEvents ObserverCategory + histogramEvents ObserverCategory lastIntervalTime time.Time } -type summaryRaw struct { +type sampleValue struct { value float64 count float64 } @@ -85,6 +102,12 @@ type summaryMetric struct { weights []float64 } +type histogramStructure = structure.Histogram[float64] + +type histogramMetric struct { + agg *histogramStructure +} + type statsDMetric struct { description statsDMetricDescription asFloat float64 @@ -113,28 +136,44 @@ func (t MetricType) FullName() TypeName { return TypeName(fmt.Sprintf("unknown(%s)", t)) } -func (p *StatsDParser) Initialize(enableMetricType bool, isMonotonicCounter bool, sendTimerHistogram []TimerHistogramMapping) error { - p.lastIntervalTime = timeNowFunc() +func (p *StatsDParser) resetState(when time.Time) { + p.lastIntervalTime = when p.gauges = make(map[statsDMetricDescription]pmetric.ScopeMetrics) p.counters = make(map[statsDMetricDescription]pmetric.ScopeMetrics) + p.timersAndDistributions = nil p.summaries = make(map[statsDMetricDescription]summaryMetric) + p.histograms = make(map[statsDMetricDescription]histogramMetric) +} - p.observeHistogram = DefaultObserverType - p.observeTimer = DefaultObserverType +func (p *StatsDParser) Initialize(enableMetricType bool, isMonotonicCounter bool, sendTimerHistogram []TimerHistogramMapping) error { + p.resetState(timeNowFunc()) + + p.histogramEvents = defaultObserverCategory + p.timerEvents = defaultObserverCategory p.enableMetricType = enableMetricType p.isMonotonicCounter = isMonotonicCounter - // Note: validation occurs in ("../".Config).vaidate() + // Note: validation occurs in ("../".Config).validate() for _, eachMap := range sendTimerHistogram { switch eachMap.StatsdType { case HistogramTypeName: - p.observeHistogram = eachMap.ObserverType + p.histogramEvents.method = eachMap.ObserverType + p.histogramEvents.histogramConfig = expoHistogramConfig(eachMap.Histogram) case TimingTypeName, TimingAltTypeName: - p.observeTimer = eachMap.ObserverType + p.timerEvents.method = eachMap.ObserverType + p.timerEvents.histogramConfig = expoHistogramConfig(eachMap.Histogram) } } return nil } +func expoHistogramConfig(opts HistogramConfig) structure.Config { + var r []structure.Option + if opts.MaxSize >= structure.MinSize { + r = append(r, structure.WithMaxSize(opts.MaxSize)) + } + return structure.NewConfig(r...) +} + // GetMetrics gets the metrics preparing for flushing and reset the state. func (p *StatsDParser) GetMetrics() pmetric.Metrics { metrics := pmetric.NewMetrics() @@ -152,34 +191,42 @@ func (p *StatsDParser) GetMetrics() pmetric.Metrics { metric.CopyTo(rm.ScopeMetrics().AppendEmpty()) } + now := timeNowFunc() + for desc, summaryMetric := range p.summaries { buildSummaryMetric( desc, summaryMetric, p.lastIntervalTime, - timeNowFunc(), + now, statsDDefaultPercentiles, rm.ScopeMetrics().AppendEmpty(), ) } - p.gauges = make(map[statsDMetricDescription]pmetric.ScopeMetrics) - p.counters = make(map[statsDMetricDescription]pmetric.ScopeMetrics) - p.timersAndDistributions = nil - p.summaries = make(map[statsDMetricDescription]summaryMetric) + for desc, histogramMetric := range p.histograms { + buildHistogramMetric( + desc, + histogramMetric, + p.lastIntervalTime, + now, + rm.ScopeMetrics().AppendEmpty(), + ) + } + p.resetState(now) return metrics } var timeNowFunc = time.Now -func (p *StatsDParser) observerTypeFor(t MetricType) ObserverType { +func (p *StatsDParser) observerCategoryFor(t MetricType) ObserverCategory { switch t { case HistogramType: - return p.observeHistogram + return p.histogramEvents case TimingType: - return p.observeTimer + return p.timerEvents } - return DisableObserver + return defaultObserverCategory } // Aggregate for each metric line. @@ -214,11 +261,12 @@ func (p *StatsDParser) Aggregate(line string) error { } case TimingType, HistogramType: - switch p.observerTypeFor(parsedMetric.description.metricType) { + category := p.observerCategoryFor(parsedMetric.description.metricType) + switch category.method { case GaugeObserver: p.timersAndDistributions = append(p.timersAndDistributions, buildGaugeMetric(parsedMetric, timeNowFunc())) case SummaryObserver: - raw := parsedMetric.summaryValue() + raw := parsedMetric.sampleValue() if existing, ok := p.summaries[parsedMetric.description]; !ok { p.summaries[parsedMetric.description] = summaryMetric{ points: []float64{raw.value}, @@ -230,6 +278,24 @@ func (p *StatsDParser) Aggregate(line string) error { weights: append(existing.weights, raw.count), } } + case HistogramObserver: + raw := parsedMetric.sampleValue() + var agg *histogramStructure + if existing, ok := p.histograms[parsedMetric.description]; ok { + agg = existing.agg + } else { + agg = new(histogramStructure) + agg.Init(category.histogramConfig) + + p.histograms[parsedMetric.description] = histogramMetric{ + agg: agg, + } + } + agg.UpdateByIncr( + raw.value, + uint64(raw.count), // Note! Rounding float64 to uint64 here. + ) + case DisableObserver: // No action. } diff --git a/receiver/statsdreceiver/protocol/statsd_parser_test.go b/receiver/statsdreceiver/protocol/statsd_parser_test.go index d01fdc4615b3d..9e98a47c728f1 100644 --- a/receiver/statsdreceiver/protocol/statsd_parser_test.go +++ b/receiver/statsdreceiver/protocol/statsd_parser_test.go @@ -19,9 +19,12 @@ import ( "testing" "time" + "github.com/lightstep/go-expohisto/mapping/logarithm" "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/otel/attribute" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/metricstestutil" ) func Test_ParseMessageToMetric(t *testing.T) { @@ -896,8 +899,8 @@ func TestStatsDParser_Initialize(t *testing.T) { attrs: *attribute.EmptySet()} p.gauges[teststatsdDMetricdescription] = pmetric.ScopeMetrics{} assert.Equal(t, 1, len(p.gauges)) - assert.Equal(t, GaugeObserver, p.observeTimer) - assert.Equal(t, GaugeObserver, p.observeHistogram) + assert.Equal(t, GaugeObserver, p.timerEvents.method) + assert.Equal(t, GaugeObserver, p.histogramEvents.method) } func TestStatsDParser_GetMetricsWithMetricType(t *testing.T) { @@ -1001,3 +1004,224 @@ func TestTimeNowFunc(t *testing.T) { timeNow := timeNowFunc() assert.NotNil(t, timeNow) } + +func TestStatsDParser_AggregateTimerWithHistogram(t *testing.T) { + timeNowFunc = func() time.Time { + return time.Unix(711, 0) + } + // It is easiest to validate data in tests such as this by limiting the + // histogram size to a small number and then setting the maximum range + // to test at scale 0, which is easy to reason about. The tests use + // max size 10, so tests w/ a range of 2**10 appear below. + normalMapping := []TimerHistogramMapping{ + { + StatsdType: "timer", + ObserverType: "histogram", + Histogram: HistogramConfig{ + MaxSize: 10, + }, + }, + { + StatsdType: "histogram", + ObserverType: "histogram", + Histogram: HistogramConfig{ + MaxSize: 10, + }, + }, + } + + newPoint := func() (pmetric.Metrics, pmetric.ExponentialHistogramDataPoint) { + data := pmetric.NewMetrics() + ilm := data.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty() + m := ilm.Metrics().AppendEmpty() + m.SetName("expohisto") + ep := m.SetEmptyExponentialHistogram() + ep.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + dp := ep.DataPoints().AppendEmpty() + + dp.Attributes().PutStr("mykey", "myvalue") + return data, dp + } + + tests := []struct { + name string + input []string + expected pmetric.Metrics + mapping []TimerHistogramMapping + }{ + { + name: "basic", + input: []string{ + "expohisto:0|ms|#mykey:myvalue", + "expohisto:1.5|ms|#mykey:myvalue", + "expohisto:2.5|ms|#mykey:myvalue", + "expohisto:4.5|ms|#mykey:myvalue", + "expohisto:8.5|ms|#mykey:myvalue", + "expohisto:16.5|ms|#mykey:myvalue", + "expohisto:32.5|ms|#mykey:myvalue", + "expohisto:64.5|ms|#mykey:myvalue", + "expohisto:128.5|ms|#mykey:myvalue", + "expohisto:256.5|ms|#mykey:myvalue", + "expohisto:512.5|ms|#mykey:myvalue", + }, + expected: func() pmetric.Metrics { + data, dp := newPoint() + dp.SetCount(11) + dp.SetSum(1028) + dp.SetMin(0) + dp.SetMax(512.5) + dp.SetZeroCount(1) + dp.SetScale(0) + dp.Positive().SetOffset(0) + dp.Positive().BucketCounts().FromRaw([]uint64{ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + }) + return data + }(), + mapping: normalMapping, + }, + { + name: "negative", + input: []string{ + "expohisto:-0|ms|#mykey:myvalue", + "expohisto:-1.5|ms|#mykey:myvalue", + "expohisto:-2.5|ms|#mykey:myvalue", + "expohisto:-4.5|ms|#mykey:myvalue", + "expohisto:-8.5|ms|#mykey:myvalue", + "expohisto:-16.5|ms|#mykey:myvalue", + "expohisto:-32.5|ms|#mykey:myvalue", + "expohisto:-64.5|ms|#mykey:myvalue", + "expohisto:-128.5|ms|#mykey:myvalue", + "expohisto:-256.5|ms|#mykey:myvalue", + "expohisto:-512.5|ms|#mykey:myvalue", + }, + expected: func() pmetric.Metrics { + data, dp := newPoint() + dp.SetCount(11) + dp.SetSum(-1028) + dp.SetMin(-512.5) + dp.SetMax(0) + dp.SetZeroCount(1) + dp.SetScale(0) + dp.Negative().SetOffset(0) + dp.Negative().BucketCounts().FromRaw([]uint64{ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + }) + return data + }(), + mapping: normalMapping, + }, + { + name: "halffull", + input: []string{ + "expohisto:1.5|ms|#mykey:myvalue", + "expohisto:4.5|ms|#mykey:myvalue", + "expohisto:16.5|ms|#mykey:myvalue", + "expohisto:64.5|ms|#mykey:myvalue", + "expohisto:512.5|ms|#mykey:myvalue", + }, + expected: func() pmetric.Metrics { + data, dp := newPoint() + dp.SetCount(5) + dp.SetSum(599.5) + dp.SetMin(1.5) + dp.SetMax(512.5) + dp.SetZeroCount(0) + dp.SetScale(0) + dp.Positive().SetOffset(0) + dp.Positive().BucketCounts().FromRaw([]uint64{ + 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, + }) + return data + }(), + mapping: normalMapping, + }, + { + name: "one_each", + input: []string{ + "expohisto:1|h|#mykey:myvalue", + "expohisto:0|h|#mykey:myvalue", + "expohisto:-1|h|#mykey:myvalue", + }, + expected: func() pmetric.Metrics { + data, dp := newPoint() + dp.SetCount(3) + dp.SetSum(0) + dp.SetMin(-1) + dp.SetMax(1) + dp.SetZeroCount(1) + dp.SetScale(logarithm.MaxScale) + dp.Positive().SetOffset(-1) + dp.Negative().SetOffset(-1) + dp.Positive().BucketCounts().FromRaw([]uint64{ + 1, + }) + dp.Negative().BucketCounts().FromRaw([]uint64{ + 1, + }) + return data + }(), + mapping: normalMapping, + }, + { + name: "all_zeros", + input: []string{ + "expohisto:0|h|#mykey:myvalue", + "expohisto:0|h|#mykey:myvalue", + "expohisto:0|h|#mykey:myvalue", + "expohisto:0|h|#mykey:myvalue", + }, + expected: func() pmetric.Metrics { + data, dp := newPoint() + dp.SetCount(4) + dp.SetSum(0) + dp.SetMin(0) + dp.SetMax(0) + dp.SetZeroCount(4) + dp.SetScale(0) + return data + }(), + mapping: normalMapping, + }, + { + name: "sampled", + input: []string{ + "expohisto:1|h|@0.125|#mykey:myvalue", + "expohisto:0|h|@0.25|#mykey:myvalue", + "expohisto:-1|h|@0.5|#mykey:myvalue", + }, + expected: func() pmetric.Metrics { + data, dp := newPoint() + dp.SetCount(14) + dp.SetSum(6) + dp.SetMin(-1) + dp.SetMax(1) + dp.SetZeroCount(4) + dp.SetScale(logarithm.MaxScale) + dp.Positive().SetOffset(-1) + dp.Negative().SetOffset(-1) + dp.Positive().BucketCounts().FromRaw([]uint64{ + 8, // 1 / 0.125 + }) + dp.Negative().BucketCounts().FromRaw([]uint64{ + 2, // 1 / 0.5 + }) + return data + }(), + mapping: normalMapping, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + p := &StatsDParser{} + assert.NoError(t, p.Initialize(false, false, tt.mapping)) + for _, line := range tt.input { + err = p.Aggregate(line) + assert.NoError(t, err) + } + var nodiffs []*metricstestutil.MetricDiff + assert.Equal(t, nodiffs, metricstestutil.DiffMetrics(nodiffs, tt.expected, p.GetMetrics())) + }) + } +} diff --git a/receiver/statsdreceiver/testdata/config.yaml b/receiver/statsdreceiver/testdata/config.yaml index 24a14e1a78505..cec588b989aac 100644 --- a/receiver/statsdreceiver/testdata/config.yaml +++ b/receiver/statsdreceiver/testdata/config.yaml @@ -8,4 +8,6 @@ statsd/receiver_settings: - statsd_type: "histogram" observer_type: "gauge" - statsd_type: "timing" - observer_type: "gauge" + observer_type: "histogram" + histogram: + max_size: 170