feat(otlp-grpc-exporter-base): use statically generated protobuf code

experimental/CHANGELOG.md

@@ -8,6 +8,8 @@ All notable changes to experimental packages in this project will be documented
 
 ### :rocket: (Enhancement)
 
+* feat(otlp-grpc-exporter-base): use statically generated protobuf code [#3705](https://github.com/open-telemetry/opentelemetry-js/pull/3705) @pichlermarc
+
 ### :bug: (Bug Fix)
 
 * fix(sdk-node): only set DiagConsoleLogger when OTEL_LOG_LEVEL is set [#3693](https://github.com/open-telemetry/opentelemetry-js/pull/3672) @pichlermarc

experimental/packages/otlp-grpc-exporter-base/.gitignore

@@ -0,0 +1,2 @@
+src/generated/*
+!src/generated/.gitkeep

experimental/packages/otlp-grpc-exporter-base/package.json

@@ -8,17 +8,16 @@
   "scripts": {
     "prepublishOnly": "npm run compile",
     "codecov": "nyc report --reporter=json && codecov -f coverage/*.json -p ../../../",
-    "compile": "tsc --build",
+    "compile": "npm run protos && tsc --build",
     "clean": "tsc --build --clean",
     "lint": "eslint . --ext .ts",
     "lint:fix": "eslint . --ext .ts --fix",
-    "postcompile": "npm run submodule && npm run protos:copy",
-    "protos:copy": "cpx protos/opentelemetry/**/*.* build/protos/opentelemetry",
+    "protos": "npm run submodule && node ../../../scripts/generate-protos.js",
     "submodule": "git submodule sync --recursive && git submodule update --init --recursive",
     "tdd": "npm run test -- --watch-extensions ts --watch",
     "test": "nyc ts-mocha -p tsconfig.json 'test/**/*.test.ts'",
     "version": "node ../../../scripts/version-update.js",
-    "watch": "npm run protos:copy && tsc -w",
+    "watch": "npm run protos && tsc -w",
     "precompile": "lerna run version --scope $(npm pkg get name) --include-dependencies",
     "prewatch": "npm run precompile"
   },
@@ -40,7 +39,6 @@
     "build/src/**/*.js",
     "build/src/**/*.js.map",
     "build/src/**/*.d.ts",
-    "build/protos/**/*.proto",
     "doc",
     "LICENSE",
     "README.md"
@@ -65,7 +63,8 @@
     "sinon": "15.0.0",
     "ts-loader": "8.4.0",
     "ts-mocha": "10.0.0",
-    "typescript": "4.4.4"
+    "typescript": "4.4.4",
+    "protobufjs-cli": "1.0.2"
   },
   "peerDependencies": {
     "@opentelemetry/api": "^1.0.0"
@@ -74,7 +73,8 @@
     "@grpc/grpc-js": "^1.7.1",
     "@grpc/proto-loader": "^0.7.3",
     "@opentelemetry/core": "1.11.0",
-    "@opentelemetry/otlp-exporter-base": "0.37.0"
+    "@opentelemetry/otlp-exporter-base": "0.37.0",
+    "protobufjs": "^7.2.2"
   },
   "homepage": "https://github.com/open-telemetry/opentelemetry-js/tree/main/experimental/packages/otlp-grpc-exporter-base",
   "sideEffects": false

experimental/packages/otlp-grpc-exporter-base/src/MetricsExportServiceClient.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as root from './generated/root';
+import * as grpc from '@grpc/grpc-js';
+import { IExportMetricsServiceRequest } from '@opentelemetry/otlp-transformer';
+import { ExportType } from './internal-types';
+import IExportMetricsServiceResponse = root.opentelemetry.proto.collector.metrics.v1.IExportMetricsServiceResponse;
+
+const responseType = root.opentelemetry.proto.collector.metrics.v1
+  .ExportMetricsServiceResponse as ExportType<IExportMetricsServiceResponse>;
+
+const requestType = root.opentelemetry.proto.collector.metrics.v1
+  .ExportMetricsServiceRequest as ExportType<IExportMetricsServiceRequest>;
+
+const metricsServiceDefinition = {
+  export: {
+    path: '/opentelemetry.proto.collector.metrics.v1.MetricsService/Export',
+    requestStream: false,
+    responseStream: false,
+    requestSerialize: (arg: IExportMetricsServiceRequest) => {
+      return Buffer.from(requestType.encode(arg).finish());
+    },
+    requestDeserialize: (arg: Buffer) => {
+      return requestType.decode(arg);
+    },
+    responseSerialize: (arg: IExportMetricsServiceResponse) => {
+      return Buffer.from(responseType.encode(arg).finish());
+    },
+    responseDeserialize: (arg: Buffer) => {
+      return responseType.decode(arg);
+    },
+  },
+};
+
+// Creates a new instance of a gRPC service client for OTLP metrics
+export const MetricExportServiceClient: grpc.ServiceClientConstructor =
+  grpc.makeGenericClientConstructor(
+    metricsServiceDefinition,
+    'MetricsExportService'
+  );

experimental/packages/otlp-grpc-exporter-base/src/TraceExportServiceClient.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as root from './generated/root';
+import * as grpc from '@grpc/grpc-js';
+import { IExportTraceServiceRequest } from '@opentelemetry/otlp-transformer';
+import { ExportType } from './internal-types';
+import IExportTraceServiceResponse = root.opentelemetry.proto.collector.trace.v1.IExportTraceServiceResponse;
+
+const responseType = root.opentelemetry.proto.collector.trace.v1
+  .ExportTraceServiceResponse as ExportType<IExportTraceServiceResponse>;
+
+const requestType = root.opentelemetry.proto.collector.trace.v1
+  .ExportTraceServiceRequest as ExportType<IExportTraceServiceRequest>;
+
+const traceServiceDefinition = {
+  export: {
+    path: '/opentelemetry.proto.collector.trace.v1.TraceService/Export',
+    requestStream: false,
+    responseStream: false,
+    requestSerialize: (arg: IExportTraceServiceRequest) => {
+      return Buffer.from(requestType.encode(arg).finish());
+    },
+    requestDeserialize: (arg: Buffer) => {
+      return requestType.decode(arg);
+    },
+    responseSerialize: (arg: IExportTraceServiceResponse) => {
+      return Buffer.from(responseType.encode(arg).finish());
+    },
+    responseDeserialize: (arg: Buffer) => {
+      return responseType.decode(arg);
+    },
+  },
+};
+
+// Creates a new instance of a gRPC service client for exporting OTLP traces
+export const TraceExportServiceClient: grpc.ServiceClientConstructor =
+  grpc.makeGenericClientConstructor(
+    traceServiceDefinition,
+    'TraceExportService'
+  );

experimental/packages/otlp-grpc-exporter-base/src/internal-types.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type * as protobuf from 'protobufjs';
+
+export interface ExportType<T, R = T & { toJSON: () => unknown }> {
+  encode(message: T, writer?: protobuf.Writer): protobuf.Writer;
+  decode(reader: protobuf.Reader | Uint8Array, length?: number): R;
+}

experimental/packages/otlp-grpc-exporter-base/src/types.ts

@@ -34,13 +34,15 @@ export interface GRPCQueueItem<ExportedItem> {
 /**
  * Service Client for sending spans or metrics
  */
-export interface ServiceClient extends grpc.Client {
+export interface ServiceClient {
   export: (
     request: any,
     metadata: grpc.Metadata,
     options: grpc.CallOptions,
     callback: Function
   ) => {};
+
+  close(): void;
 }
 
 /**

experimental/packages/otlp-grpc-exporter-base/src/util.ts

@@ -15,7 +15,6 @@
  */
 
 import * as grpc from '@grpc/grpc-js';
-import * as protoLoader from '@grpc/proto-loader';
 import { diag } from '@opentelemetry/api';
 import { getEnv, globalErrorHandler } from '@opentelemetry/core';
 import * as path from 'path';
@@ -28,11 +27,14 @@ import {
   ServiceClientType,
 } from './types';
 import {
-  CompressionAlgorithm,
   ExportServiceError,
   OTLPExporterError,
+  CompressionAlgorithm,
 } from '@opentelemetry/otlp-exporter-base';
 
+import { MetricExportServiceClient } from './MetricsExportServiceClient';
+import { TraceExportServiceClient } from './TraceExportServiceClient';
+
 export const DEFAULT_COLLECTOR_URL = 'http://localhost:4317';
 
 export function onInit<ExportItem, ServiceRequest>(
@@ -46,50 +48,33 @@ export function onInit<ExportItem, ServiceRequest>(
     collector.getUrlFromConfig(config)
   );
 
-  const includeDirs = [path.resolve(__dirname, '..', 'protos')];
-
-  protoLoader
-    .load(collector.getServiceProtoPath(), {
-      keepCase: false,
-      longs: String,
-      enums: String,
-      defaults: true,
-      oneofs: true,
-      includeDirs,
-    })
-    .then(packageDefinition => {
-      const packageObject: any = grpc.loadPackageDefinition(packageDefinition);
-
-      const options = {
-        'grpc.default_compression_algorithm': collector.compression,
-      };
+  try {
+    if (collector.getServiceClientType() === ServiceClientType.SPANS) {
+      const client = new TraceExportServiceClient(collector.url, credentials, {
+        'grpc.default_compression_algorithm': collector.compression.valueOf(),
+      });
 
-      if (collector.getServiceClientType() === ServiceClientType.SPANS) {
-        collector.serviceClient =
-          new packageObject.opentelemetry.proto.collector.trace.v1.TraceService(
-            collector.url,
-            credentials,
-            options
-          );
-      } else {
-        collector.serviceClient =
-          new packageObject.opentelemetry.proto.collector.metrics.v1.MetricsService(
-            collector.url,
-            credentials,
-            options
-          );
-      }
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      collector.serviceClient = client;
+    } else if (collector.getServiceClientType() === ServiceClientType.METRICS) {
+      const client = new MetricExportServiceClient(collector.url, credentials, {
+        'grpc.default_compression_algorithm': collector.compression.valueOf(),
+      });
 
-      if (collector.grpcQueue.length > 0) {
-        const queue = collector.grpcQueue.splice(0);
-        queue.forEach((item: GRPCQueueItem<ExportItem>) => {
-          collector.send(item.objects, item.onSuccess, item.onError);
-        });
-      }
-    })
-    .catch(err => {
-      globalErrorHandler(err);
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      collector.serviceClient = client;
+    }
+  } catch (err) {
+    globalErrorHandler(err);
+  }
+  if (collector.grpcQueue.length > 0) {
+    const queue = collector.grpcQueue.splice(0);
+    queue.forEach((item: GRPCQueueItem<ExportItem>) => {
+      collector.send(item.objects, item.onSuccess, item.onError);
     });
+  }
 }
 
 export function send<ExportItem, ServiceRequest>(

experimental/packages/otlp-grpc-exporter-base/tsconfig.json

@@ -1,12 +1,14 @@
 {
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
+    "allowJs": true,
     "outDir": "build",
     "rootDir": "."
   },
   "include": [
     "src/**/*.ts",
-    "src/generated/*.js",
+    "src/generated/**/*.js",
+    "src/generated/**/*.ts",
     "test/**/*.ts"
   ],
   "references": [

experimental/packages/otlp-proto-exporter-base/package.json

@@ -19,7 +19,7 @@
     "clean": "tsc --build --clean tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "lint": "eslint . --ext .ts",
     "lint:fix": "eslint . --ext .ts --fix",
-    "protos": "npm run submodule && node scripts/protos.js",
+    "protos": "npm run submodule && node ../../../scripts/generate-protos.js",
     "submodule": "git submodule sync --recursive && git submodule update --init --recursive",
     "version": "node ../../../scripts/version-update.js",
     "watch": "npm run protos && tsc -w tsconfig.json tsconfig.esm.json tsconfig.esnext.json",

package.json

@@ -4,14 +4,12 @@
   "description": "OpenTelemetry is a distributed tracing and stats collection framework.",
   "scripts": {
     "precompile": "lerna run version",
-    "compile": "lerna run protos && tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
+    "compile": "lerna run protos --concurrency=1 && tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "prewatch": "npm run precompile",
     "watch": "tsc --build --watch tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "clean": "tsc --build --clean tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "postinstall": "npm run update-ts-configs && npm run bootstrap",
-    "postcompile": "npm run submodule && npm run protos:copy",
     "submodule": "git submodule sync --recursive && git submodule update --init --recursive",
-    "protos:copy": "lerna run protos:copy",
     "version:update": "lerna run version:update",
     "test": "lerna run test",
     "test:browser": "lerna run test:browser",

experimental/packages/otlp-proto-exporter-base/scripts/protos.js → scripts/generate-protos.js

@@ -3,8 +3,10 @@
 const cp = require('child_process');
 const path = require('path');
 
-const generatedPath = path.resolve(__dirname, '../src/generated');
-const protosPath = path.resolve(__dirname, '../protos');
+const appRoot = process.cwd();
+
+const generatedPath = path.resolve(appRoot, './src/generated');
+const protosPath = path.resolve(appRoot, './protos');
 const protos = [
   'opentelemetry/proto/common/v1/common.proto',
   'opentelemetry/proto/resource/v1/resource.proto',
@@ -33,15 +35,15 @@ function exec(command, argv) {
 }
 
 function pbts(pbjsOutFile) {
-  const pbtsPath = path.resolve(__dirname, '../node_modules/.bin/pbts');
+  const pbtsPath = path.resolve(appRoot, './node_modules/.bin/pbts');
   const pbtsOptions = [
     '-o', path.join(generatedPath, 'root.d.ts'),
   ];
   return exec(pbtsPath, [...pbtsOptions, pbjsOutFile]);
 }
 
 async function pbjs(files) {
-  const pbjsPath = path.resolve(__dirname, '../node_modules/.bin/pbjs');
+  const pbjsPath = path.resolve(appRoot, './node_modules/.bin/pbjs');
   const outFile = path.join(generatedPath, 'root.js');
   const pbjsOptions = [
     '-t', 'static-module',