Downgrade OTel (#114)

* downgrade otel * add public key of node to metrics output * fix badge * shutdown metrics nicely
synadia-io · Feb 5, 2024 · 464b1af · 464b1af
1 parent 4d8c34b
commit 464b1af
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 52 deletions.
diff --git a/.github/workflows/ltb.yml b/.github/workflows/ltb.yml
@@ -76,10 +76,6 @@ jobs:
       name: Run test suite
       working-directory: .
       run: go test -v -race ./test
-    - 
-      name: Run spec suite
-      working-directory: .
-      run: sudo -E ~/go/bin/ginkgo run -r --randomize-all --vv --trace --keep-going --output-interceptor-mode=none ./spec
 
   build:
     timeout-minutes: 10

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-[![Lint | Test | Build](https://github.com/synadia-io/nex/actions/workflows/build.yml/badge.svg)](https://github.com/synadia-io/nex/actions/workflows/build.yml)
+[![Lint | Test | Build](https://github.com/synadia-io/nex/actions/workflows/ltb.yml/badge.svg)](https://github.com/synadia-io/nex/actions/workflows/ltb.yml)
 ![Release](https://github.com/synadia-io/nex/actions/workflows/release.yml/badge.svg)
 ![Homepage and Documentation](https://img.shields.io/website?label=Homepage&url=https%3A%2F%2Fnats.io)
 ![eg](https://img.shields.io/badge/Powered%20By-NATS-green)

diff --git a/go.mod b/go.mod
@@ -25,12 +25,12 @@ require (
 	github.com/prometheus/client_golang v1.18.0
 	github.com/rs/xid v1.5.0
 	github.com/tetratelabs/wazero v1.6.0
-	go.opentelemetry.io/otel v1.22.0
-	go.opentelemetry.io/otel/exporters/prometheus v0.45.0
-	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.45.0
-	go.opentelemetry.io/otel/metric v1.22.0
-	go.opentelemetry.io/otel/sdk v1.22.0
-	go.opentelemetry.io/otel/sdk/metric v1.22.0
+	go.opentelemetry.io/otel v1.19.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.42.0
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0
+	go.opentelemetry.io/otel/metric v1.19.0
+	go.opentelemetry.io/otel/sdk v1.19.0
+	go.opentelemetry.io/otel/sdk/metric v1.19.0
 	rogchap.com/v8go v0.9.0
 )
 
@@ -100,9 +100,9 @@ require (
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
 	go.mongodb.org/mongo-driver v1.10.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect
-	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect

diff --git a/go.sum b/go.sum
@@ -449,26 +449,26 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
-go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
-go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY=
-go.opentelemetry.io/otel/exporters/prometheus v0.45.0 h1:BeIK2KGho0oCWa7LxEGSqfDZbs7Fpv/Viz+FS4P8CXE=
-go.opentelemetry.io/otel/exporters/prometheus v0.45.0/go.mod h1:UVJZPLnfDSvHj+eJuZE+E1GjIBD267mEMfAAHJdghWg=
-go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.45.0 h1:NjN6zc7Mwy9torqa3mo+pMJ3mHoPI0uzVSYcqB2t72A=
-go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.45.0/go.mod h1:U+T5v2bk4fCC8XdSEWZja3Pm/ZhvV/zE7JwX/ELJKts=
-go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
-go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
-go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
-go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
-go.opentelemetry.io/otel/sdk/metric v1.22.0 h1:ARrRetm1HCVxq0cbnaZQlfwODYJHo3gFL8Z3tSmHBcI=
-go.opentelemetry.io/otel/sdk/metric v1.22.0/go.mod h1:KjQGeMIDlBNEOo6HvjhxIec1p/69/kULDcp4gr0oLQQ=
-go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
-go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q=
+go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
+go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
+go.opentelemetry.io/otel/exporters/prometheus v0.42.0 h1:jwV9iQdvp38fxXi8ZC+lNpxjK16MRcZlpDYvbuO1FiA=
+go.opentelemetry.io/otel/exporters/prometheus v0.42.0/go.mod h1:f3bYiqNqhoPxkvI2LrXqQVC546K7BuRDL/kKuxkujhA=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0 h1:4jJuoeOo9W6hZnz+r046fyoH5kykZPRvKfUXJVfMpB0=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0/go.mod h1:/MtYTE1SfC2QIcE0bDot6fIX+h+WvXjgTqgn9P0LNPE=
+go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
+go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o=
+go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A=
+go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k=
+go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY=
+go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
+go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

diff --git a/internal/node/node.go b/internal/node/node.go
@@ -151,7 +151,7 @@ func (n *Node) init() error {
 		err = n.loadNodeConfig()
 		// TODO-- handle this err
 
-		n.telemetry, err = NewTelemetry(n.log, n.config)
+		n.telemetry, err = NewTelemetry(n.ctx, n.log, n.config, n.publicKey)
 		if err != nil {
 			n.log.Error("Failed to initialize telemetry", slog.Any("err", err))
 			err = fmt.Errorf("failed to initialize telemetry: %s", err)
@@ -335,6 +335,7 @@ func (n *Node) shutdown() {
 		n.ncint.Close()
 		n.natsint.Shutdown()
 		n.natsint.WaitForShutdown()
+		_ = n.telemetry.Shutdown()
 	}
 }
 

diff --git a/internal/node/telemetry.go b/internal/node/telemetry.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/prometheus"
 	"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
 	"go.opentelemetry.io/otel/metric"
@@ -23,14 +24,17 @@ import (
 const defaultServiceName = "nex-node"
 
 type Telemetry struct {
-	log   *slog.Logger
-	meter metric.Meter
+	ctx           context.Context
+	log           *slog.Logger
+	meter         metric.Meter
+	meterProvider metric.MeterProvider
 
 	metricsEnabled  bool
 	metricsExporter string
 	metricsPort     int
 
 	serviceName string
+	nodePubKey  string
 
 	allocatedMemoryCounter metric.Int64UpDownCounter
 	allocatedVCPUCounter   metric.Int64UpDownCounter
@@ -44,14 +48,17 @@ type Telemetry struct {
 	functionRunTimeNano    metric.Int64Counter
 }
 
-func NewTelemetry(log *slog.Logger, config *NodeConfiguration) (*Telemetry, error) {
+func NewTelemetry(ctx context.Context, log *slog.Logger, config *NodeConfiguration, nodePubKey string) (*Telemetry, error) {
 	t := &Telemetry{
+		ctx:             ctx,
 		log:             log,
 		meter:           nil,
 		metricsEnabled:  config.OtelMetrics,
 		metricsExporter: config.OtelMetricsExporter,
 		metricsPort:     config.OtelMetricsPort,
 		serviceName:     defaultServiceName,
+		nodePubKey:      nodePubKey,
+		meterProvider:   noop.NewMeterProvider(),
 	}
 
 	err := t.init()
@@ -64,7 +71,6 @@ func NewTelemetry(log *slog.Logger, config *NodeConfiguration) (*Telemetry, erro
 
 func (t *Telemetry) init() error {
 	var e, err error
-
 	e = t.initMeterProvider()
 	if err != nil {
 		err = errors.Join(err, e)
@@ -131,45 +137,45 @@ func (t *Telemetry) init() error {
 	return err
 }
 
-func (t *Telemetry) initMeterProvider() error {
-	var meterProvider metric.MeterProvider
+func (t *Telemetry) Shutdown() error {
+	if _, ok := t.meterProvider.(*metricsdk.MeterProvider); ok {
+		return t.meterProvider.(*metricsdk.MeterProvider).Shutdown(t.ctx)
+	}
+	return nil
+}
 
+func (t *Telemetry) initMeterProvider() error {
 	if t.metricsEnabled {
+		t.log.Debug("Metrics enabled")
+
 		resource, err := resource.Merge(resource.Default(),
 			resource.NewWithAttributes(
 				semconv.SchemaURL,
 				semconv.ServiceName(t.serviceName),
 				semconv.ServiceVersion(VERSION),
+				attribute.String("node_pub_key", t.nodePubKey),
 			))
 
 		if err != nil {
 			t.log.Warn("failed to create OTel resource", slog.Any("err", err))
 			return err
 		}
 
-		metricReader, err := t.serveMetrics() // FIXME-- this seems to require some additional discussion-- it may be better suited to live in Node
+		metricReader, err := t.serveMetrics()
 		if err != nil {
 			t.log.Warn("failed to create OTel metrics exporter", slog.Any("err", err))
 			return err
 		}
 
-		meterProvider = metricsdk.NewMeterProvider(
+		t.meterProvider = metricsdk.NewMeterProvider(
 			metricsdk.WithResource(resource),
 			metricsdk.WithReader(
 				metricReader,
 			),
 		)
-
-		defer func() {
-			if err := meterProvider.(*metricsdk.MeterProvider).Shutdown(context.Background()); err != nil {
-				t.log.Error("failed to shutdown OTel meter provider", slog.Any("err", err))
-			}
-		}()
-	} else {
-		meterProvider = noop.NewMeterProvider()
 	}
 
-	otel.SetMeterProvider(meterProvider)
+	otel.SetMeterProvider(t.meterProvider)
 
 	t.meter = otel.Meter(t.serviceName)
 	if t.meter == nil {
@@ -182,6 +188,7 @@ func (t *Telemetry) initMeterProvider() error {
 func (t *Telemetry) serveMetrics() (metricsdk.Reader, error) {
 	switch t.metricsExporter {
 	case "prometheus":
+		t.log.Debug("Starting prometheus exporter")
 		go func() {
 			t.log.Info(fmt.Sprintf("serving metrics at localhost:%d/metrics", t.metricsPort))
 			http.Handle("/metrics", promhttp.Handler())
@@ -193,6 +200,7 @@ func (t *Telemetry) serveMetrics() (metricsdk.Reader, error) {
 
 		return prometheus.New()
 	default:
+		t.log.Debug("Starting standard out exporter")
 		reader, err := stdoutmetric.New()
 		if err != nil {
 			return nil, err