From 3c61b14fef87c735ea2ed382f8510b29176a4279 Mon Sep 17 00:00:00 2001 From: Blake Li Date: Mon, 22 Jan 2024 18:54:34 -0500 Subject: [PATCH] feat: Introduce interfaces for metrics instrumentation (#2403) This PR introduces new public interfaces and refactors existing interfaces for metrics instrumentation. All the interfaces are framework-agnostic. Implementation of the interfaces should use an observability framework like OpenTelemetry. --- .../google/api/gax/grpc/GrpcCallContext.java | 4 +- .../com/google/api/gax/tracing/ApiTracer.java | 38 ++++++----- .../google/api/gax/tracing/BaseApiTracer.java | 5 +- .../google/api/gax/tracing/MethodName.java | 65 +++++++++++++++++++ .../api/gax/tracing/MetricsRecorder.java | 61 +++++++++++++++++ .../google/api/gax/tracing/MetricsTracer.java | 53 +++++++++++++++ .../api/gax/tracing/MetricsTracerFactory.java | 57 ++++++++++++++++ .../java/com/google/cloud/ServiceOptions.java | 23 +++++++ 8 files changed, 287 insertions(+), 19 deletions(-) create mode 100644 gax-java/gax/src/main/java/com/google/api/gax/tracing/MethodName.java create mode 100644 gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsRecorder.java create mode 100644 gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsTracer.java create mode 100644 gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsTracerFactory.java diff --git a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcCallContext.java b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcCallContext.java index 823d8c56b7..886beda4ec 100644 --- a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcCallContext.java +++ b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcCallContext.java @@ -77,7 +77,9 @@ public final class GrpcCallContext implements ApiCallContext { private static final GrpcStatusCode UNAUTHENTICATED_STATUS_CODE = GrpcStatusCode.of(Status.Code.UNAUTHENTICATED); - static final CallOptions.Key TRACER_KEY = CallOptions.Key.create("gax.tracer"); + // This field is made public for handwritten libraries to easily access the tracer from + // CallOptions + public static final CallOptions.Key TRACER_KEY = CallOptions.Key.create("gax.tracer"); private final Channel channel; @Nullable private final Credentials credentials; diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java index 3176be4b92..6143772bac 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java @@ -49,19 +49,23 @@ public interface ApiTracer { * between clients using gax and external resources to share the same implementation of the * tracing. For example OpenCensus will install a thread local that can read by the GRPC. */ - Scope inScope(); + default Scope inScope() { + return () -> { + // noop + }; + }; /** * Signals that the overall operation has finished successfully. The tracer is now considered * closed and should no longer be used. */ - void operationSucceeded(); + default void operationSucceeded() {}; /** * Signals that the operation was cancelled by the user. The tracer is now considered closed and * should no longer be used. */ - void operationCancelled(); + default void operationCancelled() {}; /** * Signals that the overall operation has failed and no further attempts will be made. The tracer @@ -69,14 +73,14 @@ public interface ApiTracer { * * @param error the final error that caused the operation to fail. */ - void operationFailed(Throwable error); + default void operationFailed(Throwable error) {}; /** * Annotates the operation with selected connection id from the {@code ChannelPool}. * * @param id the local connection identifier of the selected connection. */ - void connectionSelected(String id); + default void connectionSelected(String id) {}; /** * Adds an annotation that an attempt is about to start. In general this should occur at the very @@ -86,7 +90,7 @@ public interface ApiTracer { * @deprecated Please use {@link #attemptStarted(Object, int)} instead. */ @Deprecated - void attemptStarted(int attemptNumber); + default void attemptStarted(int attemptNumber) {}; /** * Adds an annotation that an attempt is about to start with additional information from the @@ -96,13 +100,13 @@ public interface ApiTracer { * @param attemptNumber the zero based sequential attempt number. * @param request request of this attempt. */ - void attemptStarted(Object request, int attemptNumber); + default void attemptStarted(Object request, int attemptNumber) {}; /** Adds an annotation that the attempt succeeded. */ - void attemptSucceeded(); + default void attemptSucceeded() {}; /** Add an annotation that the attempt was cancelled by the user. */ - void attemptCancelled(); + default void attemptCancelled() {}; /** * Adds an annotation that the attempt failed, but another attempt will be made after the delay. @@ -110,7 +114,7 @@ public interface ApiTracer { * @param error the transient error that caused the attempt to fail. * @param delay the amount of time to wait before the next attempt will start. */ - void attemptFailed(Throwable error, Duration delay); + default void attemptFailed(Throwable error, Duration delay) {}; /** * Adds an annotation that the attempt failed and that no further attempts will be made because @@ -118,7 +122,7 @@ public interface ApiTracer { * * @param error the last error received before retries were exhausted. */ - void attemptFailedRetriesExhausted(Throwable error); + default void attemptFailedRetriesExhausted(Throwable error) {}; /** * Adds an annotation that the attempt failed and that no further attempts will be made because @@ -126,26 +130,26 @@ public interface ApiTracer { * * @param error the error that caused the final attempt to fail. */ - void attemptPermanentFailure(Throwable error); + default void attemptPermanentFailure(Throwable error) {}; /** * Signals that the initial RPC for the long running operation failed. * * @param error the error that caused the long running operation fail. */ - void lroStartFailed(Throwable error); + default void lroStartFailed(Throwable error) {}; /** * Signals that the initial RPC successfully started the long running operation. The long running * operation will now be polled for completion. */ - void lroStartSucceeded(); + default void lroStartSucceeded() {}; /** Adds an annotation that a streaming response has been received. */ - void responseReceived(); + default void responseReceived() {}; /** Adds an annotation that a streaming request has been sent. */ - void requestSent(); + default void requestSent() {}; /** * Adds an annotation that a batch of writes has been flushed. @@ -153,7 +157,7 @@ public interface ApiTracer { * @param elementCount the number of elements in the batch. * @param requestSize the size of the batch in bytes. */ - void batchRequestSent(long elementCount, long requestSize); + default void batchRequestSent(long elementCount, long requestSize) {}; /** * A context class to be used with {@link #inScope()} and a try-with-resources block. Closing a diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/BaseApiTracer.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/BaseApiTracer.java index 538708b879..1e542f124d 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/tracing/BaseApiTracer.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/BaseApiTracer.java @@ -33,7 +33,10 @@ import org.threeten.bp.Duration; /** - * A base implementation of {@link ApiTracer} that does nothing. + * A base implementation of {@link ApiTracer} that does nothing. With the deprecation of Java 7 + * support, all the methods in {@link ApiTracer} are now made default, we no longer need a base + * class that does nothing. This class should be removed once all the references to it are removed + * in Google Cloud Client Libraries. * *

For internal use only. */ diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/MethodName.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/MethodName.java new file mode 100644 index 0000000000..1581c0ed38 --- /dev/null +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/MethodName.java @@ -0,0 +1,65 @@ +/* + * Copyright 2024 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.tracing; + +import com.google.api.core.BetaApi; +import com.google.api.core.InternalApi; +import com.google.api.gax.rpc.StubSettings; +import com.google.auto.value.AutoValue; + +/** A value class to represent the name of the RPC method in an {@link ApiTracer}. */ +@BetaApi +@InternalApi +@AutoValue +public abstract class MethodName { + /** + * Creates a new instance of the RPC method name. + * + * @param serviceName The name of the service. In general this will be GAPIC generated service + * name {@link StubSettings#getServiceName()}. However, in some cases, when the GAPIC + * generated service is wrapped, this will be overridden to specify the manually written + * wrapper's name. + * @param methodName The name of the logical operation being traced. + */ + public static MethodName of(String serviceName, String methodName) { + return new AutoValue_MethodName(serviceName, methodName); + } + + /** The name of the service. ie BigtableData */ + public abstract String getServiceName(); + + /** The name of the logical operation being traced. ie. ReadRows. */ + public abstract String getMethodName(); + + @Override + public String toString() { + return getServiceName() + "." + getMethodName(); + } +} diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsRecorder.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsRecorder.java new file mode 100644 index 0000000000..d2e221fb5b --- /dev/null +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsRecorder.java @@ -0,0 +1,61 @@ +/* + * Copyright 2024 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.api.gax.tracing; + +import com.google.api.core.BetaApi; +import com.google.api.core.InternalApi; +import java.util.Map; + +/** + * Provides an interface for metrics recording. The implementer is expected to use an observability + * framework, e.g. OpenTelemetry. There should be only one instance of MetricsRecorder per client, + * all the methods in this class are expected to be called from multiple threads, hence the + * implementation must be thread safe. + */ +@BetaApi +@InternalApi +public interface MetricsRecorder { + + /** Records the latency of an RPC attempt */ + default void recordAttemptLatency(double attemptLatency, Map attributes) {} + + /** Records the count of RPC attempts */ + default void recordAttemptCount(long count, Map attributes) {} + + /** + * Records the total end-to-end latency for an operation, including the initial RPC attempts and + * subsequent retries. + */ + default void recordOperationLatency(double operationLatency, Map attributes) {} + + /** Records the count of operations */ + default void recordOperationCount(long count, Map attributes) {} +} diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsTracer.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsTracer.java new file mode 100644 index 0000000000..bf5dbdd046 --- /dev/null +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsTracer.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.api.gax.tracing; + +import com.google.api.core.BetaApi; +import com.google.api.core.InternalApi; + +/** + * This class computes generic metrics that can be observed in the lifecycle of an RPC operation. + * The responsibility of recording metrics should delegate to {@link MetricsRecorder}, hence this + * class should not have any knowledge about the observability framework used for metrics recording. + */ +@BetaApi +@InternalApi +public class MetricsTracer implements ApiTracer { + + public MetricsTracer(MethodName methodName, MetricsRecorder metricsRecorder) {} + + /** + * Add attributes that will be attached to all metrics. This is expected to be called by + * handwritten client teams to add additional attributes that are not supposed be collected by + * Gax. + */ + public void addAttributes(String key, String value) {}; +} diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsTracerFactory.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsTracerFactory.java new file mode 100644 index 0000000000..d2b8d87fb4 --- /dev/null +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/MetricsTracerFactory.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.tracing; + +import com.google.api.core.BetaApi; +import com.google.api.core.InternalApi; + +/** + * A {@link ApiTracerFactory} to build instances of {@link MetricsTracer}. + * + *

This class wraps the {@link MetricsRecorder} and pass it to {@link MetricsTracer}. It will be + * used to record metrics in {@link MetricsTracer}. + * + *

This class is expected to be initialized once during client initialization. + */ +@BetaApi +@InternalApi +public class MetricsTracerFactory implements ApiTracerFactory { + protected MetricsRecorder metricsRecorder; + + public MetricsTracerFactory(MetricsRecorder metricsRecorder) { + this.metricsRecorder = metricsRecorder; + } + + @Override + public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType operationType) { + return new MetricsTracer( + MethodName.of(spanName.getClientName(), spanName.getMethodName()), metricsRecorder); + } +} diff --git a/java-core/google-cloud-core/src/main/java/com/google/cloud/ServiceOptions.java b/java-core/google-cloud-core/src/main/java/com/google/cloud/ServiceOptions.java index 16879b8914..9384e7823d 100644 --- a/java-core/google-cloud-core/src/main/java/com/google/cloud/ServiceOptions.java +++ b/java-core/google-cloud-core/src/main/java/com/google/cloud/ServiceOptions.java @@ -40,6 +40,8 @@ import com.google.api.gax.rpc.FixedHeaderProvider; import com.google.api.gax.rpc.HeaderProvider; import com.google.api.gax.rpc.NoHeaderProvider; +import com.google.api.gax.tracing.ApiTracer; +import com.google.api.gax.tracing.ApiTracerFactory; import com.google.auth.Credentials; import com.google.auth.oauth2.GoogleCredentials; import com.google.auth.oauth2.QuotaProjectIdProvider; @@ -110,6 +112,8 @@ public abstract class ServiceOptions< private transient ServiceT service; private transient ServiceRpc rpc; + private final ApiTracerFactory apiTracerFactory; + /** * Builder for {@code ServiceOptions}. * @@ -137,6 +141,8 @@ public abstract static class Builder< private String clientLibToken = ServiceOptions.getGoogApiClientLibName(); private String quotaProjectId; + private ApiTracerFactory apiTracerFactory; + @InternalApi("This class should only be extended within google-cloud-java") protected Builder() {} @@ -153,6 +159,7 @@ protected Builder(ServiceOptions options) { transportOptions = options.transportOptions; clientLibToken = options.clientLibToken; quotaProjectId = options.quotaProjectId; + apiTracerFactory = options.apiTracerFactory; } protected abstract ServiceOptions build(); @@ -306,6 +313,17 @@ public B setQuotaProjectId(String quotaProjectId) { return self(); } + /** + * Sets the {@link ApiTracerFactory}. It will be used to create an {@link ApiTracer} that is + * annotated throughout the lifecycle of an RPC operation. + */ + @BetaApi + @InternalApi + public B setApiTracerFactory(ApiTracerFactory apiTracerFactory) { + this.apiTracerFactory = apiTracerFactory; + return self(); + } + protected Set getAllowedClientLibTokens() { return allowedClientLibTokens; } @@ -347,6 +365,7 @@ protected ServiceOptions( builder.quotaProjectId != null ? builder.quotaProjectId : getValueFromCredentialsFile(getCredentialsPath(), "quota_project_id"); + apiTracerFactory = builder.apiTracerFactory; } private static String getCredentialsPath() { @@ -692,6 +711,10 @@ public String getLibraryVersion() { return GaxProperties.getLibraryVersion(this.getClass()); } + public ApiTracerFactory getApiTracerFactory() { + return apiTracerFactory; + } + @InternalApi public final HeaderProvider getMergedHeaderProvider(HeaderProvider internalHeaderProvider) { Map mergedHeaders =