Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Downgrade OTel #114

Merged
merged 6 commits into from Feb 5, 2024
Merged

Downgrade OTel #114

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a4f42a3
downgrade otel
jordan-rash Feb 5, 2024
bb09c46
add public key of node to metrics output
jordan-rash Feb 5, 2024
25c2222
rider - fix badge
jordan-rash Feb 5, 2024
8a6a956
jordan needs a babysitter
jordan-rash Feb 5, 2024
d3166a4
shutdown metrics nicely
jordan-rash Feb 5, 2024
8a438f1
shutdown func
jordan-rash Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 0 additions & 4 deletions .github/workflows/ltb.yml
View file
Open in desktop
Expand Up @@ -76,10 +76,6 @@ jobs:
name: Run test suite
working-directory: .
run: go test -v -race ./test
-
name: Run spec suite
working-directory: .
run: sudo -E ~/go/bin/ginkgo run -r --randomize-all --vv --trace --keep-going --output-interceptor-mode=none ./spec

build:
timeout-minutes: 10
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
@@ -1,4 +1,4 @@
[![Lint | Test | Build](https://github.com/synadia-io/nex/actions/workflows/build.yml/badge.svg)](https://github.com/synadia-io/nex/actions/workflows/build.yml)
[![Lint | Test | Build](https://github.com/synadia-io/nex/actions/workflows/ltb.yml/badge.svg)](https://github.com/synadia-io/nex/actions/workflows/ltb.yml)
![Release](https://github.com/synadia-io/nex/actions/workflows/release.yml/badge.svg)
![Homepage and Documentation](https://img.shields.io/website?label=Homepage&url=https%3A%2F%2Fnats.io)
![eg](https://img.shields.io/badge/Powered%20By-NATS-green)
Expand Down
18 changes: 9 additions & 9 deletions go.mod
View file
Open in desktop
Expand Up @@ -25,12 +25,12 @@ require (
github.com/prometheus/client_golang v1.18.0
github.com/rs/xid v1.5.0
github.com/tetratelabs/wazero v1.6.0
go.opentelemetry.io/otel v1.22.0
go.opentelemetry.io/otel/exporters/prometheus v0.45.0
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.45.0
go.opentelemetry.io/otel/metric v1.22.0
go.opentelemetry.io/otel/sdk v1.22.0
go.opentelemetry.io/otel/sdk/metric v1.22.0
go.opentelemetry.io/otel v1.19.0
go.opentelemetry.io/otel/exporters/prometheus v0.42.0
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0
go.opentelemetry.io/otel/metric v1.19.0
go.opentelemetry.io/otel/sdk v1.19.0
go.opentelemetry.io/otel/sdk/metric v1.19.0
rogchap.com/v8go v0.9.0
)

Expand Down Expand Up @@ -100,9 +100,9 @@ require (
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
go.mongodb.org/mongo-driver v1.10.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect
go.opentelemetry.io/otel/trace v1.22.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
go.uber.org/goleak v1.3.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
Expand Down
40 changes: 20 additions & 20 deletions go.sum
View file
Open in desktop
Expand Up @@ -449,26 +449,26 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY=
go.opentelemetry.io/otel/exporters/prometheus v0.45.0 h1:BeIK2KGho0oCWa7LxEGSqfDZbs7Fpv/Viz+FS4P8CXE=
go.opentelemetry.io/otel/exporters/prometheus v0.45.0/go.mod h1:UVJZPLnfDSvHj+eJuZE+E1GjIBD267mEMfAAHJdghWg=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.45.0 h1:NjN6zc7Mwy9torqa3mo+pMJ3mHoPI0uzVSYcqB2t72A=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.45.0/go.mod h1:U+T5v2bk4fCC8XdSEWZja3Pm/ZhvV/zE7JwX/ELJKts=
go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
go.opentelemetry.io/otel/sdk/metric v1.22.0 h1:ARrRetm1HCVxq0cbnaZQlfwODYJHo3gFL8Z3tSmHBcI=
go.opentelemetry.io/otel/sdk/metric v1.22.0/go.mod h1:KjQGeMIDlBNEOo6HvjhxIec1p/69/kULDcp4gr0oLQQ=
go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q=
go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
go.opentelemetry.io/otel/exporters/prometheus v0.42.0 h1:jwV9iQdvp38fxXi8ZC+lNpxjK16MRcZlpDYvbuO1FiA=
go.opentelemetry.io/otel/exporters/prometheus v0.42.0/go.mod h1:f3bYiqNqhoPxkvI2LrXqQVC546K7BuRDL/kKuxkujhA=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0 h1:4jJuoeOo9W6hZnz+r046fyoH5kykZPRvKfUXJVfMpB0=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0/go.mod h1:/MtYTE1SfC2QIcE0bDot6fIX+h+WvXjgTqgn9P0LNPE=
go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o=
go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A=
go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k=
go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY=
go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
Expand Down
3 changes: 2 additions & 1 deletion internal/node/node.go
View file
Open in desktop
Expand Up @@ -151,7 +151,7 @@ func (n *Node) init() error {
err = n.loadNodeConfig()
// TODO-- handle this err

n.telemetry, err = NewTelemetry(n.log, n.config)
n.telemetry, err = NewTelemetry(n.log, n.config, n.publicKey)
if err != nil {
n.log.Error("Failed to initialize telemetry", slog.Any("err", err))
err = fmt.Errorf("failed to initialize telemetry: %s", err)
Expand Down Expand Up @@ -335,6 +335,7 @@ func (n *Node) shutdown() {
n.ncint.Close()
n.natsint.Shutdown()
n.natsint.WaitForShutdown()
_ = n.telemetry.providerShutdown(n.ctx)
jordan-rash marked this conversation as resolved.
Show resolved Hide resolved
}
}

Expand Down
40 changes: 24 additions & 16 deletions internal/node/telemetry.go
View file
Open in desktop
Expand Up @@ -9,6 +9,7 @@ import (
"time"

"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
"go.opentelemetry.io/otel/metric"
Expand All @@ -23,14 +24,16 @@ import (
const defaultServiceName = "nex-node"

type Telemetry struct {
log *slog.Logger
meter metric.Meter
log *slog.Logger
meter metric.Meter
providerShutdown func(context.Context) error

metricsEnabled bool
metricsExporter string
metricsPort int

serviceName string
nodePubKey string

allocatedMemoryCounter metric.Int64UpDownCounter
allocatedVCPUCounter metric.Int64UpDownCounter
Expand All @@ -44,14 +47,16 @@ type Telemetry struct {
functionRunTimeNano metric.Int64Counter
}

func NewTelemetry(log *slog.Logger, config *NodeConfiguration) (*Telemetry, error) {
func NewTelemetry(log *slog.Logger, config *NodeConfiguration, nodePubKey string) (*Telemetry, error) {
jordan-rash marked this conversation as resolved.
Show resolved Hide resolved
t := &Telemetry{
log: log,
meter: nil,
metricsEnabled: config.OtelMetrics,
metricsExporter: config.OtelMetricsExporter,
metricsPort: config.OtelMetricsPort,
serviceName: defaultServiceName,
log: log,
meter: nil,
metricsEnabled: config.OtelMetrics,
metricsExporter: config.OtelMetricsExporter,
metricsPort: config.OtelMetricsPort,
serviceName: defaultServiceName,
nodePubKey: nodePubKey,
providerShutdown: func(_ context.Context) error { return nil },
}

err := t.init()
Expand All @@ -64,7 +69,6 @@ func NewTelemetry(log *slog.Logger, config *NodeConfiguration) (*Telemetry, erro

func (t *Telemetry) init() error {
var e, err error

e = t.initMeterProvider()
if err != nil {
err = errors.Join(err, e)
Expand Down Expand Up @@ -135,19 +139,22 @@ func (t *Telemetry) initMeterProvider() error {
var meterProvider metric.MeterProvider

if t.metricsEnabled {
t.log.Debug("Metrics enabled")

resource, err := resource.Merge(resource.Default(),
resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceName(t.serviceName),
semconv.ServiceVersion(VERSION),
attribute.String("node_pub_key", t.nodePubKey),
))

if err != nil {
t.log.Warn("failed to create OTel resource", slog.Any("err", err))
return err
}

metricReader, err := t.serveMetrics() // FIXME-- this seems to require some additional discussion-- it may be better suited to live in Node
metricReader, err := t.serveMetrics()
if err != nil {
t.log.Warn("failed to create OTel metrics exporter", slog.Any("err", err))
return err
Expand All @@ -160,11 +167,10 @@ func (t *Telemetry) initMeterProvider() error {
),
)

defer func() {
if err := meterProvider.(*metricsdk.MeterProvider).Shutdown(context.Background()); err != nil {
t.log.Error("failed to shutdown OTel meter provider", slog.Any("err", err))
}
}()
if _, ok := meterProvider.(*metricsdk.MeterProvider); ok {
t.providerShutdown = meterProvider.(*metricsdk.MeterProvider).Shutdown
}

} else {
meterProvider = noop.NewMeterProvider()
jordan-rash marked this conversation as resolved.
Show resolved Hide resolved
}
Expand All @@ -182,6 +188,7 @@ func (t *Telemetry) initMeterProvider() error {
func (t *Telemetry) serveMetrics() (metricsdk.Reader, error) {
switch t.metricsExporter {
case "prometheus":
t.log.Debug("Starting prometheus exporter")
go func() {
t.log.Info(fmt.Sprintf("serving metrics at localhost:%d/metrics", t.metricsPort))
http.Handle("/metrics", promhttp.Handler())
Expand All @@ -193,6 +200,7 @@ func (t *Telemetry) serveMetrics() (metricsdk.Reader, error) {

return prometheus.New()
default:
t.log.Debug("Starting standard out exporter")
reader, err := stdoutmetric.New()
if err != nil {
return nil, err
Expand Down