From 4c60e21985b43b512e12d2a1575634d4e0613563 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Tue, 28 Sep 2021 13:03:19 -0400 Subject: [PATCH] Add ability to parse Schema files according to OTEP 0152 The parser and parsed representation (AST) are placed in a separate Go module so that they are can be consumed independently without the need to bring the rest of the SDK. Ability to use the parsed representation for schema conversions can be added later. --- .github/dependabot.yml | 11 ++ bridge/opencensus/go.mod | 2 + bridge/opencensus/test/go.mod | 2 + bridge/opentracing/go.mod | 2 + example/fib/go.mod | 2 + example/jaeger/go.mod | 2 + example/namedtracer/go.mod | 2 + example/opencensus/go.mod | 2 + example/otel-collector/go.mod | 2 + example/passthrough/go.mod | 2 + example/prometheus/go.mod | 2 + example/zipkin/go.mod | 2 + exporters/jaeger/go.mod | 2 + exporters/otlp/otlpmetric/go.mod | 2 + .../otlp/otlpmetric/otlpmetricgrpc/go.mod | 2 + .../otlp/otlpmetric/otlpmetrichttp/go.mod | 2 + exporters/otlp/otlptrace/go.mod | 2 + exporters/otlp/otlptrace/otlptracegrpc/go.mod | 2 + exporters/otlp/otlptrace/otlptracehttp/go.mod | 2 + exporters/prometheus/go.mod | 2 + exporters/stdout/stdoutmetric/go.mod | 2 + exporters/stdout/stdouttrace/go.mod | 2 + exporters/zipkin/go.mod | 2 + go.mod | 2 + internal/metric/go.mod | 2 + internal/tools/go.mod | 2 + metric/go.mod | 2 + schema/ast/ast_schema.go | 63 ++++++++ schema/ast/logs.go | 30 ++++ schema/ast/metrics.go | 34 +++++ schema/ast/spans.go | 54 +++++++ schema/go.mod | 71 +++++++++ schema/go.sum | 26 ++++ schema/parser.go | 94 ++++++++++++ schema/parser_test.go | 48 ++++++ schema/testdata/invalid-schema-url.yaml | 5 + schema/testdata/unsupported-file-format.yaml | 9 ++ schema/testdata/valid-example.yaml | 138 ++++++++++++++++++ schema/types/types.go | 21 +++ sdk/export/metric/go.mod | 2 + sdk/go.mod | 2 + sdk/metric/go.mod | 2 + trace/go.mod | 2 + 43 files changed, 664 insertions(+) create mode 100644 schema/ast/ast_schema.go create mode 100644 schema/ast/logs.go create mode 100644 schema/ast/metrics.go create mode 100644 schema/ast/spans.go create mode 100644 schema/go.mod create mode 100644 schema/go.sum create mode 100644 schema/parser.go create mode 100644 schema/parser_test.go create mode 100644 schema/testdata/invalid-schema-url.yaml create mode 100644 schema/testdata/unsupported-file-format.yaml create mode 100644 schema/testdata/valid-example.yaml create mode 100644 schema/types/types.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 86cf1f19185..d17e5855df2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -336,3 +336,14 @@ updates: schedule: day: sunday interval: weekly + + - + package-ecosystem: gomod + directory: /schema + labels: + - dependencies + - go + - "Skip Changelog" + schedule: + day: sunday + interval: weekly diff --git a/bridge/opencensus/go.mod b/bridge/opencensus/go.mod index 47d4add96f0..0905be5b580 100644 --- a/bridge/opencensus/go.mod +++ b/bridge/opencensus/go.mod @@ -73,3 +73,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ./test replace go.opentelemetry.io/otel/example/fib => ../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/bridge/opencensus/test/go.mod b/bridge/opencensus/test/go.mod index 35c1b21302c..b5d9860634e 100644 --- a/bridge/opencensus/test/go.mod +++ b/bridge/opencensus/test/go.mod @@ -69,3 +69,5 @@ replace go.opentelemetry.io/otel/sdk/metric => ../../../sdk/metric replace go.opentelemetry.io/otel/trace => ../../../trace replace go.opentelemetry.io/otel/example/fib => ../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../schema diff --git a/bridge/opentracing/go.mod b/bridge/opentracing/go.mod index 18fba8aaa92..128fb89c2dd 100644 --- a/bridge/opentracing/go.mod +++ b/bridge/opentracing/go.mod @@ -69,3 +69,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/example/fib/go.mod b/example/fib/go.mod index 7990c3ceb5d..67b06ebe739 100644 --- a/example/fib/go.mod +++ b/example/fib/go.mod @@ -68,3 +68,5 @@ replace go.opentelemetry.io/otel/sdk/metric => ../../sdk/metric replace go.opentelemetry.io/otel/trace => ../../trace replace go.opentelemetry.io/otel/example/fib => ./ + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/example/jaeger/go.mod b/example/jaeger/go.mod index 3818e5177e9..634aff4178c 100644 --- a/example/jaeger/go.mod +++ b/example/jaeger/go.mod @@ -69,3 +69,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/example/namedtracer/go.mod b/example/namedtracer/go.mod index 321c6b6f511..cbcba5aa0f1 100644 --- a/example/namedtracer/go.mod +++ b/example/namedtracer/go.mod @@ -71,3 +71,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/example/opencensus/go.mod b/example/opencensus/go.mod index 258cbcbd785..e6a62edbb33 100644 --- a/example/opencensus/go.mod +++ b/example/opencensus/go.mod @@ -73,3 +73,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/example/otel-collector/go.mod b/example/otel-collector/go.mod index 55bc79b3094..601a383c6b3 100644 --- a/example/otel-collector/go.mod +++ b/example/otel-collector/go.mod @@ -72,3 +72,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/example/passthrough/go.mod b/example/passthrough/go.mod index 04ed842d6f1..74b6ebf807f 100644 --- a/example/passthrough/go.mod +++ b/example/passthrough/go.mod @@ -72,3 +72,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/example/prometheus/go.mod b/example/prometheus/go.mod index 0cd1397f1a7..bd48753875d 100644 --- a/example/prometheus/go.mod +++ b/example/prometheus/go.mod @@ -71,3 +71,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/example/zipkin/go.mod b/example/zipkin/go.mod index 23876b1dbec..41849679bd0 100644 --- a/example/zipkin/go.mod +++ b/example/zipkin/go.mod @@ -70,3 +70,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/exporters/jaeger/go.mod b/exporters/jaeger/go.mod index 42d337d5495..15696b80908 100644 --- a/exporters/jaeger/go.mod +++ b/exporters/jaeger/go.mod @@ -73,3 +73,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/exporters/otlp/otlpmetric/go.mod b/exporters/otlp/otlpmetric/go.mod index fc8cb13f94b..2dcb197db70 100644 --- a/exporters/otlp/otlpmetric/go.mod +++ b/exporters/otlp/otlpmetric/go.mod @@ -80,3 +80,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ./o replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../schema diff --git a/exporters/otlp/otlpmetric/otlpmetricgrpc/go.mod b/exporters/otlp/otlpmetric/otlpmetricgrpc/go.mod index 7851b6ce907..684484bfe6e 100644 --- a/exporters/otlp/otlpmetric/otlpmetricgrpc/go.mod +++ b/exporters/otlp/otlpmetric/otlpmetricgrpc/go.mod @@ -78,3 +78,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../../schema diff --git a/exporters/otlp/otlpmetric/otlpmetrichttp/go.mod b/exporters/otlp/otlpmetric/otlpmetrichttp/go.mod index 74202836e62..372d5fece1d 100644 --- a/exporters/otlp/otlpmetric/otlpmetrichttp/go.mod +++ b/exporters/otlp/otlpmetric/otlpmetrichttp/go.mod @@ -80,3 +80,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../../schema diff --git a/exporters/otlp/otlptrace/go.mod b/exporters/otlp/otlptrace/go.mod index 6812e6eb71d..60bb12fcfa6 100644 --- a/exporters/otlp/otlptrace/go.mod +++ b/exporters/otlp/otlptrace/go.mod @@ -76,3 +76,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../schema diff --git a/exporters/otlp/otlptrace/otlptracegrpc/go.mod b/exporters/otlp/otlptrace/otlptracegrpc/go.mod index 636c98333dc..97b1dd6c9a0 100644 --- a/exporters/otlp/otlptrace/otlptracegrpc/go.mod +++ b/exporters/otlp/otlptrace/otlptracegrpc/go.mod @@ -72,3 +72,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../../schema diff --git a/exporters/otlp/otlptrace/otlptracehttp/go.mod b/exporters/otlp/otlptrace/otlptracehttp/go.mod index 52b34857764..d7ef48e6f39 100644 --- a/exporters/otlp/otlptrace/otlptracehttp/go.mod +++ b/exporters/otlp/otlptrace/otlptracehttp/go.mod @@ -73,3 +73,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../../schema diff --git a/exporters/prometheus/go.mod b/exporters/prometheus/go.mod index 1aea41a0581..ee89c821c82 100644 --- a/exporters/prometheus/go.mod +++ b/exporters/prometheus/go.mod @@ -75,3 +75,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/exporters/stdout/stdoutmetric/go.mod b/exporters/stdout/stdoutmetric/go.mod index ccbd8bfa5f6..264d3eb1c6a 100644 --- a/exporters/stdout/stdoutmetric/go.mod +++ b/exporters/stdout/stdoutmetric/go.mod @@ -73,3 +73,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../schema diff --git a/exporters/stdout/stdouttrace/go.mod b/exporters/stdout/stdouttrace/go.mod index 9db2350f8a8..e4ac2183a47 100644 --- a/exporters/stdout/stdouttrace/go.mod +++ b/exporters/stdout/stdouttrace/go.mod @@ -71,3 +71,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../schema diff --git a/exporters/zipkin/go.mod b/exporters/zipkin/go.mod index 33235f25b36..5042ef694a0 100644 --- a/exporters/zipkin/go.mod +++ b/exporters/zipkin/go.mod @@ -74,3 +74,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/go.mod b/go.mod index 447addf6bfd..b6e0a6fd946 100644 --- a/go.mod +++ b/go.mod @@ -69,3 +69,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ./e replace go.opentelemetry.io/otel/bridge/opencensus/test => ./bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ./example/fib + +replace go.opentelemetry.io/otel/schema => ./schema diff --git a/internal/metric/go.mod b/internal/metric/go.mod index ee661d43d0a..af3e09dfe7a 100644 --- a/internal/metric/go.mod +++ b/internal/metric/go.mod @@ -69,3 +69,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 57bc409f55b..046b168032f 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -75,3 +75,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/metric/go.mod b/metric/go.mod index e8134901afb..e921d01b6b9 100644 --- a/metric/go.mod +++ b/metric/go.mod @@ -70,3 +70,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../example/fib + +replace go.opentelemetry.io/otel/schema => ../schema diff --git a/schema/ast/ast_schema.go b/schema/ast/ast_schema.go new file mode 100644 index 00000000000..61f69f825d8 --- /dev/null +++ b/schema/ast/ast_schema.go @@ -0,0 +1,63 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ast // import "go.opentelemetry.io/otel/schema/ast" + +import "go.opentelemetry.io/otel/schema/types" + +const FileFormat = "1.0.0" + +// Schema represents a Schema file in FileFormat 1.0.0 as defined in +// https://github.com/open-telemetry/oteps/blob/main/text/0152-telemetry-schemas.md +type Schema struct { + // Schema file format. SHOULD be 1.0.0 for the current specification version. + // See https://github.com/open-telemetry/oteps/blob/main/text/0152-telemetry-schemas.md#schema-file-format-number + FileFormat string `yaml:"file_format"` + + // Schema URL is an identifier of a Schema. The URL specifies a location of this + // Schema File that can be retrieved (so it is a URL and not just a URI) using HTTP + // or HTTPS protocol. + // See https://github.com/open-telemetry/oteps/blob/main/text/0152-telemetry-schemas.md#schema-url + SchemaURL string `yaml:"schema_url"` + + // Versions section that lists changes that happened in each particular version. + Versions map[types.TelemetryVersion]VersionDef +} + +// VersionDef corresponds to a section representing one version under the "versions" +// top-level key. +type VersionDef struct { + All VersionOfAttributes + Resources VersionOfAttributes + Spans VersionOfSpans + SpanEvents VersionOfSpanEvents `yaml:"span_events"` + Logs VersionOfLogs + Metrics VersionOfMetrics +} + +// VersionOfAttributes corresponds to a section representing a list of changes that +// happened in a particular version. +type VersionOfAttributes struct { + Changes []AttributeChanges +} + +// AttributeChanges corresponds to a section representing attribute changes. +type AttributeChanges struct { + RenameAttributes *MappingOfAttributes `yaml:"rename_attributes"` +} + +// MappingOfAttributes corresponds to a section representing a mapping of attribute names. +// The keys are the old attribute name used the previous version, the values are the +// new attribute name starting from this version. +type MappingOfAttributes map[string]string diff --git a/schema/ast/logs.go b/schema/ast/logs.go new file mode 100644 index 00000000000..9d977c6b295 --- /dev/null +++ b/schema/ast/logs.go @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ast // import "go.opentelemetry.io/otel/schema/ast" + +// VersionOfLogs corresponds to a section representing a list of changes that happened +// to logs schema in a particular version. +type VersionOfLogs struct { + Changes []LogsChange +} + +// LogsChange corresponds to a section representing logs change. +type LogsChange struct { + RenameAttributes *RenameLogAttributes `yaml:"rename_attributes"` +} + +type RenameLogAttributes struct { + AttributeMap map[string]string `yaml:"attribute_map"` +} diff --git a/schema/ast/metrics.go b/schema/ast/metrics.go new file mode 100644 index 00000000000..b06a2170e6e --- /dev/null +++ b/schema/ast/metrics.go @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ast + +import "go.opentelemetry.io/otel/schema/types" + +// VersionOfMetrics corresponds to a section representing a list of changes that happened +// to metrics schema in a particular version. +type VersionOfMetrics struct { + Changes []MetricsChange +} + +// MetricsChange corresponds to a section representing metrics change. +type MetricsChange struct { + RenameMetrics map[types.MetricName]types.MetricName `yaml:"rename_metrics"` + RenameLabels *LabelMapForMetrics `yaml:"rename_labels"` +} + +type LabelMapForMetrics struct { + ApplyToMetrics []types.MetricName `yaml:"apply_to_metrics"` + LabelMap map[string]string `yaml:"label_map"` +} diff --git a/schema/ast/spans.go b/schema/ast/spans.go new file mode 100644 index 00000000000..0d714a04e1a --- /dev/null +++ b/schema/ast/spans.go @@ -0,0 +1,54 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ast // import "go.opentelemetry.io/otel/schema/ast" + +import "go.opentelemetry.io/otel/schema/types" + +// VersionOfSpans corresponds to a section representing a list of changes that happened +// to spans schema in a particular version. +type VersionOfSpans struct { + Changes []SpansChange +} + +// VersionOfSpanEvents corresponds to a section representing a list of changes that happened +// to span events schema in a particular version. +type VersionOfSpanEvents struct { + Changes []SpanEventsChange +} + +// SpansChange corresponds to a section representing spans change. +type SpansChange struct { + RenameAttributes *RenameSpanAttributes `yaml:"rename_attributes"` +} + +// SpanEventsChange corresponds to a section representing span events change. +type SpanEventsChange struct { + RenameEvents *RenameSpanEvents `yaml:"rename_events"` + RenameAttributes *RenameSpanEventAttributes `yaml:"rename_attributes"` +} + +type RenameSpanAttributes struct { + AttributeMap map[string]string `yaml:"attribute_map"` +} + +type RenameSpanEvents struct { + EventNameMap map[string]string `yaml:"name_map"` +} + +type RenameSpanEventAttributes struct { + ApplyToSpans []types.SpanName `yaml:"apply_to_spans"` + ApplyToEvents []types.EventName `yaml:"apply_to_events"` + AttributeMap map[string]string `yaml:"attribute_map"` +} diff --git a/schema/go.mod b/schema/go.mod new file mode 100644 index 00000000000..82c4602cb9d --- /dev/null +++ b/schema/go.mod @@ -0,0 +1,71 @@ +module go.opentelemetry.io/otel/schema + +go 1.15 + +require ( + github.com/stretchr/testify v1.7.0 + golang.org/x/mod v0.5.1 + gopkg.in/yaml.v2 v2.4.0 +) + +replace go.opentelemetry.io/otel => ../ + +replace go.opentelemetry.io/otel/bridge/opencensus => ../bridge/opencensus + +replace go.opentelemetry.io/otel/bridge/opencensus/test => ../bridge/opencensus/test + +replace go.opentelemetry.io/otel/bridge/opentracing => ../bridge/opentracing + +replace go.opentelemetry.io/otel/example/fib => ../example/fib + +replace go.opentelemetry.io/otel/example/jaeger => ../example/jaeger + +replace go.opentelemetry.io/otel/example/namedtracer => ../example/namedtracer + +replace go.opentelemetry.io/otel/example/opencensus => ../example/opencensus + +replace go.opentelemetry.io/otel/example/otel-collector => ../example/otel-collector + +replace go.opentelemetry.io/otel/example/passthrough => ../example/passthrough + +replace go.opentelemetry.io/otel/example/prometheus => ../example/prometheus + +replace go.opentelemetry.io/otel/example/zipkin => ../example/zipkin + +replace go.opentelemetry.io/otel/exporters/jaeger => ../exporters/jaeger + +replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric => ../exporters/otlp/otlpmetric + +replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc => ../exporters/otlp/otlpmetric/otlpmetricgrpc + +replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../exporters/otlp/otlpmetric/otlpmetrichttp + +replace go.opentelemetry.io/otel/exporters/otlp/otlptrace => ../exporters/otlp/otlptrace + +replace go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => ../exporters/otlp/otlptrace/otlptracegrpc + +replace go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp => ../exporters/otlp/otlptrace/otlptracehttp + +replace go.opentelemetry.io/otel/exporters/prometheus => ../exporters/prometheus + +replace go.opentelemetry.io/otel/exporters/stdout/stdoutmetric => ../exporters/stdout/stdoutmetric + +replace go.opentelemetry.io/otel/exporters/stdout/stdouttrace => ../exporters/stdout/stdouttrace + +replace go.opentelemetry.io/otel/exporters/zipkin => ../exporters/zipkin + +replace go.opentelemetry.io/otel/internal/metric => ../internal/metric + +replace go.opentelemetry.io/otel/internal/tools => ../internal/tools + +replace go.opentelemetry.io/otel/metric => ../metric + +replace go.opentelemetry.io/otel/schema => ./ + +replace go.opentelemetry.io/otel/sdk => ../sdk + +replace go.opentelemetry.io/otel/sdk/export/metric => ../sdk/export/metric + +replace go.opentelemetry.io/otel/sdk/metric => ../sdk/metric + +replace go.opentelemetry.io/otel/trace => ../trace diff --git a/schema/go.sum b/schema/go.sum new file mode 100644 index 00000000000..553ed1c8b0a --- /dev/null +++ b/schema/go.sum @@ -0,0 +1,26 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/schema/parser.go b/schema/parser.go new file mode 100644 index 00000000000..5ddc96a9b6b --- /dev/null +++ b/schema/parser.go @@ -0,0 +1,94 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema // import "go.opentelemetry.io/otel/schema" + +import ( + "fmt" + "io/ioutil" + "net/url" + "strings" + + "golang.org/x/mod/semver" + "gopkg.in/yaml.v2" + + "go.opentelemetry.io/otel/schema/ast" +) + +func Parse(schemaFile string) (*ast.Schema, error) { + var ts ast.Schema + schemaContent, err := ioutil.ReadFile(schemaFile) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(schemaContent, &ts) + if err != nil { + return nil, err + } + + if err := checkFileFormatField(ts.FileFormat); err != nil { + return nil, err + } + + if strings.TrimSpace(ts.SchemaURL) == "" { + return nil, fmt.Errorf("schema_url field is missing") + } + + if _, err := url.Parse(ts.SchemaURL); err != nil { + return nil, fmt.Errorf("invalid URL specified in schema_url field: %v", err) + } + + return &ts, nil +} + +// checkFileFormatField validates the file format field according to the rules here: +// https://github.com/open-telemetry/oteps/blob/main/text/0152-telemetry-schemas.md#schema-file-format-number +func checkFileFormatField(fileFormat string) error { + if !semver.IsValid("v" + fileFormat) { + return fmt.Errorf( + "invalud schema file format version number %q (expected semver)", + fileFormat, + ) + } + + // Check that the major version number is the same as what we expect. + expected := semver.Major("v" + ast.FileFormat) + got := semver.Major("v" + fileFormat) + if expected != got { + expected = strings.TrimLeft(expected, "v") + got = strings.TrimLeft(got, "v") + return fmt.Errorf( + "unsupported schema file format major version number, expected %v, got %v", + expected, got, + ) + } + + // Check that the minor version number is not newer than what we expect. + expected = semver.MajorMinor("v" + ast.FileFormat) + + got = semver.MajorMinor("v" + fileFormat) + if expected != got && semver.Compare(expected, got) < 0 { + expected = strings.TrimLeft(expected, "v") + ".x" + got = fileFormat + return fmt.Errorf( + "unsupported schema file format minor version number, expected no newer than %v, got %v", + expected, got, + ) + } + + // Patch version number does not matter, so we don't check it. + + return nil +} diff --git a/schema/parser_test.go b/schema/parser_test.go new file mode 100644 index 00000000000..180db33dcc2 --- /dev/null +++ b/schema/parser_test.go @@ -0,0 +1,48 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/otel/schema/ast" +) + +func TestParseSchema(t *testing.T) { + ts, err := Parse("testdata/valid-example.yaml") + require.NoError(t, err) + require.NotNil(t, ts) +} + +func TestFailParseSchema(t *testing.T) { + ts, err := Parse("testdata/unsupported-file-format.yaml") + require.Error(t, err) + require.Nil(t, ts) + + ts, err = Parse("testdata/invalid-schema-url.yaml") + require.Error(t, err) + require.Nil(t, ts) +} + +func TestCheckFileFormatField(t *testing.T) { + assert.Error(t, checkFileFormatField("not a semver")) + assert.Error(t, checkFileFormatField("2.0.0")) + assert.Error(t, checkFileFormatField("1.1.0")) + assert.NoError(t, checkFileFormatField("1.0.1")) + assert.NoError(t, checkFileFormatField(ast.FileFormat)) +} diff --git a/schema/testdata/invalid-schema-url.yaml b/schema/testdata/invalid-schema-url.yaml new file mode 100644 index 00000000000..8a9001b205d --- /dev/null +++ b/schema/testdata/invalid-schema-url.yaml @@ -0,0 +1,5 @@ +file_format: 1.0.0 + +schema_url: http://invalid url + +versions: diff --git a/schema/testdata/unsupported-file-format.yaml b/schema/testdata/unsupported-file-format.yaml new file mode 100644 index 00000000000..fb24f4861a8 --- /dev/null +++ b/schema/testdata/unsupported-file-format.yaml @@ -0,0 +1,9 @@ +file_format: 1.1.0 + +versions: + 1.1.0: + all: + changes: + - rename_attributes: + k8s.cluster.name: kubernetes.cluster.name + 1.0.0: diff --git a/schema/testdata/valid-example.yaml b/schema/testdata/valid-example.yaml new file mode 100644 index 00000000000..283def97232 --- /dev/null +++ b/schema/testdata/valid-example.yaml @@ -0,0 +1,138 @@ +file_format: 1.0.0 + +schema_url: https://opentelemetry.io/schemas/1.1.0 + +versions: + 1.1.0: + # Section "all" applies to attributes names for all data types: resources, spans, logs, + # span events, metric labels. + # + # The translations in "all" section are performed first (for each particular version). + # Only after that the translations in the specific section ("resources", "traces", + # "metrics" or "logs") that corresponds to the data type are applied. + # + # The only translation possible in section "all" is renaming of attributes in + # versions. For human readability versions are listed in reverse chronological + # order, however note that the translations are applied in the order defined by + # semver ordering. + all: + changes: + - rename_attributes: + # Mapping of attribute names (label names for metrics). The key is the old name + # used prior to this version, the value is the new name starting from this version. + + # Rename k8s.* to kubernetes.* + k8s.cluster.name: kubernetes.cluster.name + k8s.namespace.name: kubernetes.namespace.name + k8s.node.name: kubernetes.node.name + k8s.node.uid: kubernetes.node.uid + k8s.pod.name: kubernetes.pod.name + k8s.pod.uid: kubernetes.pod.uid + k8s.container.name: kubernetes.container.name + k8s.replicaset.name: kubernetes.replicaset.name + k8s.replicaset.uid: kubernetes.replicaset.uid + k8s.cronjob.name: kubernetes.cronjob.name + k8s.cronjob.uid: kubernetes.cronjob.uid + k8s.job.name: kubernetes.job.name + k8s.job.uid: kubernetes.job.uid + k8s.statefulset.name: kubernetes.statefulset.name + k8s.statefulset.uid: kubernetes.statefulset.uid + k8s.daemonset.name: kubernetes.daemonset.name + k8s.daemonset.uid: kubernetes.daemonset.uid + k8s.deployment.name: kubernetes.deployment.name + k8s.deployment.uid: kubernetes.deployment.uid + + service.namespace: service.namespace.name + + # Like "all" the "resources" section may contain only attribute renaming translations. + # The only translation possible in this section is renaming of attributes in + # versions. + resources: + changes: + - rename_attributes: + # Mapping of attribute names. The key is the old name + # used prior to this version, the value is the new name starting from this version. + telemetry.auto.version: telemetry.auto_instr.version + + spans: + changes: + # Sequence of translations to apply to convert the schema from a prior version + # to this version. The order in this sequence is important. Translations are + # applied from top to bottom in the listed order. + - rename_attributes: + # Rename attributes of all spans, regardless of span name. + # The keys are the old attribute name used prior to this version, the values are + # the new attribute name starting from this version. + attribute_map: + peer.service: peer.service.name + + span_events: + changes: + # Sequence of translations to apply to convert the schema from a prior version + # to this version. The order in this sequence is important. Translations are + # applied from top to bottom in the listed order. + - rename_events: + # Rename events. The keys are old event names, the values are the new event names. + name_map: {exception.stacktrace: exception.stack_trace} + + - rename_attributes: + # Rename attributes of events. + # The keys are the old attribute name used prior to this version, the values are + # the new attribute name starting from this version. + attribute_map: + peer.service: peer.service.name + + apply_to_events: + # Optional event names to apply to. If empty applies to all events. + # Conditions in apply_to_spans and apply_to_events are logical AND-ed, + # both should match for transformation to be applied. + - exception.stack_trace + + metrics: + changes: + # Sequence of translations to apply to convert the schema from a prior version + # to this version. The order in this sequence is important. Translations are + # applied from top to bottom in the listed order. + + - rename_labels: + # Rename labels of all metrics, regardless of metric name. + # The keys are the old label name used prior to this version, the values are + # the new label name starting from this version. + label_map: + http.status_code: http.response_status_code + + - rename_metrics: + # Rename metrics. The keys are old metric names, the values are the new metric names. + container.cpu.usage.total: cpu.usage.total + container.memory.usage.max: memory.usage.max + + - rename_labels: + apply_to_metrics: + # Name of the metric to apply this rule to. If empty the rule applies to all metrics. + - system.cpu.utilization + - system.memory.usage + - system.memory.utilization + - system.paging.usage + label_map: + # The keys are the old label name used prior to this version, the values are + # the new label name starting from this version. + status: state + + logs: + changes: + - rename_logs: + # map of key/values. The keys are the old log name used + # the previous version, the values are the new log name + # starting from this version. + ProcessStarted: otel.process.started + - rename_attributes: + attribute_map: + process.executable_name: process.executable.name + apply_to_logs: + # Optional. If it is missing the transformation is applied + # to all logs. If it is present the transformation is applied + # only to the logs with the name that is found in the sequence + # specified below. + - otel.process.started + + 1.0.0: diff --git a/schema/types/types.go b/schema/types/types.go new file mode 100644 index 00000000000..d5a59ab48f1 --- /dev/null +++ b/schema/types/types.go @@ -0,0 +1,21 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types // import "go.opentelemetry.io/otel/schema/types" + +type TelemetryVersion string + +type SpanName string +type EventName string +type MetricName string diff --git a/sdk/export/metric/go.mod b/sdk/export/metric/go.mod index c55ae3cc297..59421f36689 100644 --- a/sdk/export/metric/go.mod +++ b/sdk/export/metric/go.mod @@ -70,3 +70,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../../schema diff --git a/sdk/go.mod b/sdk/go.mod index 575bb4f0f89..2d74f8af89f 100644 --- a/sdk/go.mod +++ b/sdk/go.mod @@ -71,3 +71,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../example/fib + +replace go.opentelemetry.io/otel/schema => ../schema diff --git a/sdk/metric/go.mod b/sdk/metric/go.mod index d10ac30e2c3..e662a503e10 100644 --- a/sdk/metric/go.mod +++ b/sdk/metric/go.mod @@ -73,3 +73,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../../example/fib + +replace go.opentelemetry.io/otel/schema => ../../schema diff --git a/trace/go.mod b/trace/go.mod index fac348c9ff1..ade03c1a3c8 100644 --- a/trace/go.mod +++ b/trace/go.mod @@ -69,3 +69,5 @@ replace go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp => ../ replace go.opentelemetry.io/otel/bridge/opencensus/test => ../bridge/opencensus/test replace go.opentelemetry.io/otel/example/fib => ../example/fib + +replace go.opentelemetry.io/otel/schema => ../schema