From 72a256f3059d903bca37ba27826278a0bd0400a9 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Mon, 20 Jul 2020 13:42:57 -0700 Subject: [PATCH 1/9] xds: import v3 proto for ADS service --- .../v3/AggregatedDiscoveryServiceGrpc.java | 376 ++++++++++++++++++ xds/third_party/envoy/import.sh | 2 + .../envoy/service/discovery/v3/ads.proto | 44 ++ .../service/discovery/v3/discovery.proto | 244 ++++++++++++ 4 files changed, 666 insertions(+) create mode 100644 xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java create mode 100644 xds/third_party/envoy/src/main/proto/envoy/service/discovery/v3/ads.proto create mode 100644 xds/third_party/envoy/src/main/proto/envoy/service/discovery/v3/discovery.proto diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java b/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java new file mode 100644 index 00000000000..a2698b43853 --- /dev/null +++ b/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java @@ -0,0 +1,376 @@ +package io.envoyproxy.envoy.service.discovery.v3; + +import static io.grpc.MethodDescriptor.generateFullMethodName; +import static io.grpc.stub.ClientCalls.asyncBidiStreamingCall; +import static io.grpc.stub.ClientCalls.asyncClientStreamingCall; +import static io.grpc.stub.ClientCalls.asyncServerStreamingCall; +import static io.grpc.stub.ClientCalls.asyncUnaryCall; +import static io.grpc.stub.ClientCalls.blockingServerStreamingCall; +import static io.grpc.stub.ClientCalls.blockingUnaryCall; +import static io.grpc.stub.ClientCalls.futureUnaryCall; +import static io.grpc.stub.ServerCalls.asyncBidiStreamingCall; +import static io.grpc.stub.ServerCalls.asyncClientStreamingCall; +import static io.grpc.stub.ServerCalls.asyncServerStreamingCall; +import static io.grpc.stub.ServerCalls.asyncUnaryCall; +import static io.grpc.stub.ServerCalls.asyncUnimplementedStreamingCall; +import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; + +/** + * 
+ * See https://github.com/lyft/envoy-api#apis for a description of the role of
+ * ADS and how it is intended to be used by a management server. ADS requests
+ * have the same structure as their singleton xDS counterparts, but can
+ * multiplex many resource types on a single stream. The type_url in the
+ * DiscoveryRequest/DiscoveryResponse provides sufficient information to recover
+ * the multiplexed singleton APIs at the Envoy instance and management server.
+ *
+ */ +@javax.annotation.Generated( + value = "by gRPC proto compiler", + comments = "Source: envoy/service/discovery/v3/ads.proto") +public final class AggregatedDiscoveryServiceGrpc { + + private AggregatedDiscoveryServiceGrpc() {} + + public static final String SERVICE_NAME = "envoy.service.discovery.v3.AggregatedDiscoveryService"; + + // Static method descriptors that strictly reflect the proto. + private static volatile io.grpc.MethodDescriptor getStreamAggregatedResourcesMethod; + + @io.grpc.stub.annotations.RpcMethod( + fullMethodName = SERVICE_NAME + '/' + "StreamAggregatedResources", + requestType = io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest.class, + responseType = io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse.class, + methodType = io.grpc.MethodDescriptor.MethodType.BIDI_STREAMING) + public static io.grpc.MethodDescriptor getStreamAggregatedResourcesMethod() { + io.grpc.MethodDescriptor getStreamAggregatedResourcesMethod; + if ((getStreamAggregatedResourcesMethod = AggregatedDiscoveryServiceGrpc.getStreamAggregatedResourcesMethod) == null) { + synchronized (AggregatedDiscoveryServiceGrpc.class) { + if ((getStreamAggregatedResourcesMethod = AggregatedDiscoveryServiceGrpc.getStreamAggregatedResourcesMethod) == null) { + AggregatedDiscoveryServiceGrpc.getStreamAggregatedResourcesMethod = getStreamAggregatedResourcesMethod = + io.grpc.MethodDescriptor.newBuilder() + .setType(io.grpc.MethodDescriptor.MethodType.BIDI_STREAMING) + .setFullMethodName(generateFullMethodName(SERVICE_NAME, "StreamAggregatedResources")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest.getDefaultInstance())) + .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( + io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse.getDefaultInstance())) + .setSchemaDescriptor(new AggregatedDiscoveryServiceMethodDescriptorSupplier("StreamAggregatedResources")) + .build(); + } + } + } + return getStreamAggregatedResourcesMethod; + } + + private static volatile io.grpc.MethodDescriptor getDeltaAggregatedResourcesMethod; + + @io.grpc.stub.annotations.RpcMethod( + fullMethodName = SERVICE_NAME + '/' + "DeltaAggregatedResources", + requestType = io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest.class, + responseType = io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse.class, + methodType = io.grpc.MethodDescriptor.MethodType.BIDI_STREAMING) + public static io.grpc.MethodDescriptor getDeltaAggregatedResourcesMethod() { + io.grpc.MethodDescriptor getDeltaAggregatedResourcesMethod; + if ((getDeltaAggregatedResourcesMethod = AggregatedDiscoveryServiceGrpc.getDeltaAggregatedResourcesMethod) == null) { + synchronized (AggregatedDiscoveryServiceGrpc.class) { + if ((getDeltaAggregatedResourcesMethod = AggregatedDiscoveryServiceGrpc.getDeltaAggregatedResourcesMethod) == null) { + AggregatedDiscoveryServiceGrpc.getDeltaAggregatedResourcesMethod = getDeltaAggregatedResourcesMethod = + io.grpc.MethodDescriptor.newBuilder() + .setType(io.grpc.MethodDescriptor.MethodType.BIDI_STREAMING) + .setFullMethodName(generateFullMethodName(SERVICE_NAME, "DeltaAggregatedResources")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( + io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest.getDefaultInstance())) + .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( + io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse.getDefaultInstance())) + .setSchemaDescriptor(new AggregatedDiscoveryServiceMethodDescriptorSupplier("DeltaAggregatedResources")) + .build(); + } + } + } + return getDeltaAggregatedResourcesMethod; + } + + /** + * Creates a new async stub that supports all call types for the service + */ + public static AggregatedDiscoveryServiceStub newStub(io.grpc.Channel channel) { + io.grpc.stub.AbstractStub.StubFactory factory = + new io.grpc.stub.AbstractStub.StubFactory() { + @java.lang.Override + public AggregatedDiscoveryServiceStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AggregatedDiscoveryServiceStub(channel, callOptions); + } + }; + return AggregatedDiscoveryServiceStub.newStub(factory, channel); + } + + /** + * Creates a new blocking-style stub that supports unary and streaming output calls on the service + */ + public static AggregatedDiscoveryServiceBlockingStub newBlockingStub( + io.grpc.Channel channel) { + io.grpc.stub.AbstractStub.StubFactory factory = + new io.grpc.stub.AbstractStub.StubFactory() { + @java.lang.Override + public AggregatedDiscoveryServiceBlockingStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AggregatedDiscoveryServiceBlockingStub(channel, callOptions); + } + }; + return AggregatedDiscoveryServiceBlockingStub.newStub(factory, channel); + } + + /** + * Creates a new ListenableFuture-style stub that supports unary calls on the service + */ + public static AggregatedDiscoveryServiceFutureStub newFutureStub( + io.grpc.Channel channel) { + io.grpc.stub.AbstractStub.StubFactory factory = + new io.grpc.stub.AbstractStub.StubFactory() { + @java.lang.Override + public AggregatedDiscoveryServiceFutureStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AggregatedDiscoveryServiceFutureStub(channel, callOptions); + } + }; + return AggregatedDiscoveryServiceFutureStub.newStub(factory, channel); + } + + /** + * 
+   * See https://github.com/lyft/envoy-api#apis for a description of the role of
+   * ADS and how it is intended to be used by a management server. ADS requests
+   * have the same structure as their singleton xDS counterparts, but can
+   * multiplex many resource types on a single stream. The type_url in the
+   * DiscoveryRequest/DiscoveryResponse provides sufficient information to recover
+   * the multiplexed singleton APIs at the Envoy instance and management server.
+   *
+ */ + public static abstract class AggregatedDiscoveryServiceImplBase implements io.grpc.BindableService { + + /** + * 
+     * This is a gRPC-only API.
+     *
+ */ + public io.grpc.stub.StreamObserver streamAggregatedResources( + io.grpc.stub.StreamObserver responseObserver) { + return asyncUnimplementedStreamingCall(getStreamAggregatedResourcesMethod(), responseObserver); + } + + /** + */ + public io.grpc.stub.StreamObserver deltaAggregatedResources( + io.grpc.stub.StreamObserver responseObserver) { + return asyncUnimplementedStreamingCall(getDeltaAggregatedResourcesMethod(), responseObserver); + } + + @java.lang.Override public final io.grpc.ServerServiceDefinition bindService() { + return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor()) + .addMethod( + getStreamAggregatedResourcesMethod(), + asyncBidiStreamingCall( + new MethodHandlers< + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse>( + this, METHODID_STREAM_AGGREGATED_RESOURCES))) + .addMethod( + getDeltaAggregatedResourcesMethod(), + asyncBidiStreamingCall( + new MethodHandlers< + io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest, + io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse>( + this, METHODID_DELTA_AGGREGATED_RESOURCES))) + .build(); + } + } + + /** + * 
+   * See https://github.com/lyft/envoy-api#apis for a description of the role of
+   * ADS and how it is intended to be used by a management server. ADS requests
+   * have the same structure as their singleton xDS counterparts, but can
+   * multiplex many resource types on a single stream. The type_url in the
+   * DiscoveryRequest/DiscoveryResponse provides sufficient information to recover
+   * the multiplexed singleton APIs at the Envoy instance and management server.
+   *
+ */ + public static final class AggregatedDiscoveryServiceStub extends io.grpc.stub.AbstractAsyncStub { + private AggregatedDiscoveryServiceStub( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @java.lang.Override + protected AggregatedDiscoveryServiceStub build( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AggregatedDiscoveryServiceStub(channel, callOptions); + } + + /** + * 
+     * This is a gRPC-only API.
+     *
+ */ + public io.grpc.stub.StreamObserver streamAggregatedResources( + io.grpc.stub.StreamObserver responseObserver) { + return asyncBidiStreamingCall( + getChannel().newCall(getStreamAggregatedResourcesMethod(), getCallOptions()), responseObserver); + } + + /** + */ + public io.grpc.stub.StreamObserver deltaAggregatedResources( + io.grpc.stub.StreamObserver responseObserver) { + return asyncBidiStreamingCall( + getChannel().newCall(getDeltaAggregatedResourcesMethod(), getCallOptions()), responseObserver); + } + } + + /** + * 
+   * See https://github.com/lyft/envoy-api#apis for a description of the role of
+   * ADS and how it is intended to be used by a management server. ADS requests
+   * have the same structure as their singleton xDS counterparts, but can
+   * multiplex many resource types on a single stream. The type_url in the
+   * DiscoveryRequest/DiscoveryResponse provides sufficient information to recover
+   * the multiplexed singleton APIs at the Envoy instance and management server.
+   *
+ */ + public static final class AggregatedDiscoveryServiceBlockingStub extends io.grpc.stub.AbstractBlockingStub { + private AggregatedDiscoveryServiceBlockingStub( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @java.lang.Override + protected AggregatedDiscoveryServiceBlockingStub build( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AggregatedDiscoveryServiceBlockingStub(channel, callOptions); + } + } + + /** + * 
+   * See https://github.com/lyft/envoy-api#apis for a description of the role of
+   * ADS and how it is intended to be used by a management server. ADS requests
+   * have the same structure as their singleton xDS counterparts, but can
+   * multiplex many resource types on a single stream. The type_url in the
+   * DiscoveryRequest/DiscoveryResponse provides sufficient information to recover
+   * the multiplexed singleton APIs at the Envoy instance and management server.
+   *
+ */ + public static final class AggregatedDiscoveryServiceFutureStub extends io.grpc.stub.AbstractFutureStub { + private AggregatedDiscoveryServiceFutureStub( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @java.lang.Override + protected AggregatedDiscoveryServiceFutureStub build( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AggregatedDiscoveryServiceFutureStub(channel, callOptions); + } + } + + private static final int METHODID_STREAM_AGGREGATED_RESOURCES = 0; + private static final int METHODID_DELTA_AGGREGATED_RESOURCES = 1; + + private static final class MethodHandlers implements + io.grpc.stub.ServerCalls.UnaryMethod, + io.grpc.stub.ServerCalls.ServerStreamingMethod, + io.grpc.stub.ServerCalls.ClientStreamingMethod, + io.grpc.stub.ServerCalls.BidiStreamingMethod { + private final AggregatedDiscoveryServiceImplBase serviceImpl; + private final int methodId; + + MethodHandlers(AggregatedDiscoveryServiceImplBase serviceImpl, int methodId) { + this.serviceImpl = serviceImpl; + this.methodId = methodId; + } + + @java.lang.Override + @java.lang.SuppressWarnings("unchecked") + public void invoke(Req request, io.grpc.stub.StreamObserver responseObserver) { + switch (methodId) { + default: + throw new AssertionError(); + } + } + + @java.lang.Override + @java.lang.SuppressWarnings("unchecked") + public io.grpc.stub.StreamObserver invoke( + io.grpc.stub.StreamObserver responseObserver) { + switch (methodId) { + case METHODID_STREAM_AGGREGATED_RESOURCES: + return (io.grpc.stub.StreamObserver) serviceImpl.streamAggregatedResources( + (io.grpc.stub.StreamObserver) responseObserver); + case METHODID_DELTA_AGGREGATED_RESOURCES: + return (io.grpc.stub.StreamObserver) serviceImpl.deltaAggregatedResources( + (io.grpc.stub.StreamObserver) responseObserver); + default: + throw new AssertionError(); + } + } + } + + private static abstract class AggregatedDiscoveryServiceBaseDescriptorSupplier + implements io.grpc.protobuf.ProtoFileDescriptorSupplier, io.grpc.protobuf.ProtoServiceDescriptorSupplier { + AggregatedDiscoveryServiceBaseDescriptorSupplier() {} + + @java.lang.Override + public com.google.protobuf.Descriptors.FileDescriptor getFileDescriptor() { + return io.envoyproxy.envoy.service.discovery.v3.AdsProto.getDescriptor(); + } + + @java.lang.Override + public com.google.protobuf.Descriptors.ServiceDescriptor getServiceDescriptor() { + return getFileDescriptor().findServiceByName("AggregatedDiscoveryService"); + } + } + + private static final class AggregatedDiscoveryServiceFileDescriptorSupplier + extends AggregatedDiscoveryServiceBaseDescriptorSupplier { + AggregatedDiscoveryServiceFileDescriptorSupplier() {} + } + + private static final class AggregatedDiscoveryServiceMethodDescriptorSupplier + extends AggregatedDiscoveryServiceBaseDescriptorSupplier + implements io.grpc.protobuf.ProtoMethodDescriptorSupplier { + private final String methodName; + + AggregatedDiscoveryServiceMethodDescriptorSupplier(String methodName) { + this.methodName = methodName; + } + + @java.lang.Override + public com.google.protobuf.Descriptors.MethodDescriptor getMethodDescriptor() { + return getServiceDescriptor().findMethodByName(methodName); + } + } + + private static volatile io.grpc.ServiceDescriptor serviceDescriptor; + + public static io.grpc.ServiceDescriptor getServiceDescriptor() { + io.grpc.ServiceDescriptor result = serviceDescriptor; + if (result == null) { + synchronized (AggregatedDiscoveryServiceGrpc.class) { + result = serviceDescriptor; + if (result == null) { + serviceDescriptor = result = io.grpc.ServiceDescriptor.newBuilder(SERVICE_NAME) + .setSchemaDescriptor(new AggregatedDiscoveryServiceFileDescriptorSupplier()) + .addMethod(getStreamAggregatedResourcesMethod()) + .addMethod(getDeltaAggregatedResourcesMethod()) + .build(); + } + } + } + return result; + } +} diff --git a/xds/third_party/envoy/import.sh b/xds/third_party/envoy/import.sh index e528611d171..cd16468c50b 100755 --- a/xds/third_party/envoy/import.sh +++ b/xds/third_party/envoy/import.sh @@ -117,6 +117,8 @@ envoy/extensions/transport_sockets/tls/v3/secret.proto envoy/extensions/transport_sockets/tls/v3/tls.proto envoy/service/discovery/v2/ads.proto envoy/service/discovery/v2/sds.proto +envoy/service/discovery/v3/ads.proto +envoy/service/discovery/v3/discovery.proto envoy/service/load_stats/v2/lrs.proto envoy/type/http.proto envoy/type/matcher/regex.proto diff --git a/xds/third_party/envoy/src/main/proto/envoy/service/discovery/v3/ads.proto b/xds/third_party/envoy/src/main/proto/envoy/service/discovery/v3/ads.proto new file mode 100644 index 00000000000..03021559ab6 --- /dev/null +++ b/xds/third_party/envoy/src/main/proto/envoy/service/discovery/v3/ads.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; + +package envoy.service.discovery.v3; + +import "envoy/service/discovery/v3/discovery.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.service.discovery.v3"; +option java_outer_classname = "AdsProto"; +option java_multiple_files = true; +option java_generic_services = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Aggregated Discovery Service (ADS)] + +// [#not-implemented-hide:] Discovery services for endpoints, clusters, routes, +// and listeners are retained in the package `envoy.api.v2` for backwards +// compatibility with existing management servers. New development in discovery +// services should proceed in the package `envoy.service.discovery.v2`. + +// See https://github.com/lyft/envoy-api#apis for a description of the role of +// ADS and how it is intended to be used by a management server. ADS requests +// have the same structure as their singleton xDS counterparts, but can +// multiplex many resource types on a single stream. The type_url in the +// DiscoveryRequest/DiscoveryResponse provides sufficient information to recover +// the multiplexed singleton APIs at the Envoy instance and management server. +service AggregatedDiscoveryService { + // This is a gRPC-only API. + rpc StreamAggregatedResources(stream DiscoveryRequest) returns (stream DiscoveryResponse) { + } + + rpc DeltaAggregatedResources(stream DeltaDiscoveryRequest) + returns (stream DeltaDiscoveryResponse) { + } +} + +// [#not-implemented-hide:] Not configuration. Workaround c++ protobuf issue with importing +// services: https://github.com/google/protobuf/issues/4221 +message AdsDummy { + option (udpa.annotations.versioning).previous_message_type = + "envoy.service.discovery.v2.AdsDummy"; +} diff --git a/xds/third_party/envoy/src/main/proto/envoy/service/discovery/v3/discovery.proto b/xds/third_party/envoy/src/main/proto/envoy/service/discovery/v3/discovery.proto new file mode 100644 index 00000000000..b8e31160a88 --- /dev/null +++ b/xds/third_party/envoy/src/main/proto/envoy/service/discovery/v3/discovery.proto @@ -0,0 +1,244 @@ +syntax = "proto3"; + +package envoy.service.discovery.v3; + +import "envoy/config/core/v3/base.proto"; + +import "google/protobuf/any.proto"; +import "google/rpc/status.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.service.discovery.v3"; +option java_outer_classname = "DiscoveryProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Common discovery API components] + +// A DiscoveryRequest requests a set of versioned resources of the same type for +// a given Envoy node on some API. +// [#next-free-field: 7] +message DiscoveryRequest { + option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.DiscoveryRequest"; + + // The version_info provided in the request messages will be the version_info + // received with the most recent successfully processed response or empty on + // the first request. It is expected that no new request is sent after a + // response is received until the Envoy instance is ready to ACK/NACK the new + // configuration. ACK/NACK takes place by returning the new API config version + // as applied or the previous API config version respectively. Each type_url + // (see below) has an independent version associated with it. + string version_info = 1; + + // The node making the request. + config.core.v3.Node node = 2; + + // List of resources to subscribe to, e.g. list of cluster names or a route + // configuration name. If this is empty, all resources for the API are + // returned. LDS/CDS may have empty resource_names, which will cause all + // resources for the Envoy instance to be returned. The LDS and CDS responses + // will then imply a number of resources that need to be fetched via EDS/RDS, + // which will be explicitly enumerated in resource_names. + repeated string resource_names = 3; + + // Type of the resource that is being requested, e.g. + // "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment". This is implicit + // in requests made via singleton xDS APIs such as CDS, LDS, etc. but is + // required for ADS. + string type_url = 4; + + // nonce corresponding to DiscoveryResponse being ACK/NACKed. See above + // discussion on version_info and the DiscoveryResponse nonce comment. This + // may be empty only if 1) this is a non-persistent-stream xDS such as HTTP, + // or 2) the client has not yet accepted an update in this xDS stream (unlike + // delta, where it is populated only for new explicit ACKs). + string response_nonce = 5; + + // This is populated when the previous :ref:`DiscoveryResponse ` + // failed to update configuration. The *message* field in *error_details* provides the Envoy + // internal exception related to the failure. It is only intended for consumption during manual + // debugging, the string provided is not guaranteed to be stable across Envoy versions. + google.rpc.Status error_detail = 6; +} + +// [#next-free-field: 7] +message DiscoveryResponse { + option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.DiscoveryResponse"; + + // The version of the response data. + string version_info = 1; + + // The response resources. These resources are typed and depend on the API being called. + repeated google.protobuf.Any resources = 2; + + // [#not-implemented-hide:] + // Canary is used to support two Envoy command line flags: + // + // * --terminate-on-canary-transition-failure. When set, Envoy is able to + // terminate if it detects that configuration is stuck at canary. Consider + // this example sequence of updates: + // - Management server applies a canary config successfully. + // - Management server rolls back to a production config. + // - Envoy rejects the new production config. + // Since there is no sensible way to continue receiving configuration + // updates, Envoy will then terminate and apply production config from a + // clean slate. + // * --dry-run-canary. When set, a canary response will never be applied, only + // validated via a dry run. + bool canary = 3; + + // Type URL for resources. Identifies the xDS API when muxing over ADS. + // Must be consistent with the type_url in the 'resources' repeated Any (if non-empty). + string type_url = 4; + + // For gRPC based subscriptions, the nonce provides a way to explicitly ack a + // specific DiscoveryResponse in a following DiscoveryRequest. Additional + // messages may have been sent by Envoy to the management server for the + // previous version on the stream prior to this DiscoveryResponse, that were + // unprocessed at response send time. The nonce allows the management server + // to ignore any further DiscoveryRequests for the previous version until a + // DiscoveryRequest bearing the nonce. The nonce is optional and is not + // required for non-stream based xDS implementations. + string nonce = 5; + + // [#not-implemented-hide:] + // The control plane instance that sent the response. + config.core.v3.ControlPlane control_plane = 6; +} + +// DeltaDiscoveryRequest and DeltaDiscoveryResponse are used in a new gRPC +// endpoint for Delta xDS. +// +// With Delta xDS, the DeltaDiscoveryResponses do not need to include a full +// snapshot of the tracked resources. Instead, DeltaDiscoveryResponses are a +// diff to the state of a xDS client. +// In Delta XDS there are per-resource versions, which allow tracking state at +// the resource granularity. +// An xDS Delta session is always in the context of a gRPC bidirectional +// stream. This allows the xDS server to keep track of the state of xDS clients +// connected to it. +// +// In Delta xDS the nonce field is required and used to pair +// DeltaDiscoveryResponse to a DeltaDiscoveryRequest ACK or NACK. +// Optionally, a response message level system_version_info is present for +// debugging purposes only. +// +// DeltaDiscoveryRequest plays two independent roles. Any DeltaDiscoveryRequest +// can be either or both of: [1] informing the server of what resources the +// client has gained/lost interest in (using resource_names_subscribe and +// resource_names_unsubscribe), or [2] (N)ACKing an earlier resource update from +// the server (using response_nonce, with presence of error_detail making it a NACK). +// Additionally, the first message (for a given type_url) of a reconnected gRPC stream +// has a third role: informing the server of the resources (and their versions) +// that the client already possesses, using the initial_resource_versions field. +// +// As with state-of-the-world, when multiple resource types are multiplexed (ADS), +// all requests/acknowledgments/updates are logically walled off by type_url: +// a Cluster ACK exists in a completely separate world from a prior Route NACK. +// In particular, initial_resource_versions being sent at the "start" of every +// gRPC stream actually entails a message for each type_url, each with its own +// initial_resource_versions. +// [#next-free-field: 8] +message DeltaDiscoveryRequest { + option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.DeltaDiscoveryRequest"; + + // The node making the request. + config.core.v3.Node node = 1; + + // Type of the resource that is being requested, e.g. + // "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment". + string type_url = 2; + + // DeltaDiscoveryRequests allow the client to add or remove individual + // resources to the set of tracked resources in the context of a stream. + // All resource names in the resource_names_subscribe list are added to the + // set of tracked resources and all resource names in the resource_names_unsubscribe + // list are removed from the set of tracked resources. + // + // *Unlike* state-of-the-world xDS, an empty resource_names_subscribe or + // resource_names_unsubscribe list simply means that no resources are to be + // added or removed to the resource list. + // *Like* state-of-the-world xDS, the server must send updates for all tracked + // resources, but can also send updates for resources the client has not subscribed to. + // + // NOTE: the server must respond with all resources listed in resource_names_subscribe, + // even if it believes the client has the most recent version of them. The reason: + // the client may have dropped them, but then regained interest before it had a chance + // to send the unsubscribe message. See DeltaSubscriptionStateTest.RemoveThenAdd. + // + // These two fields can be set in any DeltaDiscoveryRequest, including ACKs + // and initial_resource_versions. + // + // A list of Resource names to add to the list of tracked resources. + repeated string resource_names_subscribe = 3; + + // A list of Resource names to remove from the list of tracked resources. + repeated string resource_names_unsubscribe = 4; + + // Informs the server of the versions of the resources the xDS client knows of, to enable the + // client to continue the same logical xDS session even in the face of gRPC stream reconnection. + // It will not be populated: [1] in the very first stream of a session, since the client will + // not yet have any resources, [2] in any message after the first in a stream (for a given + // type_url), since the server will already be correctly tracking the client's state. + // (In ADS, the first message *of each type_url* of a reconnected stream populates this map.) + // The map's keys are names of xDS resources known to the xDS client. + // The map's values are opaque resource versions. + map initial_resource_versions = 5; + + // When the DeltaDiscoveryRequest is a ACK or NACK message in response + // to a previous DeltaDiscoveryResponse, the response_nonce must be the + // nonce in the DeltaDiscoveryResponse. + // Otherwise (unlike in DiscoveryRequest) response_nonce must be omitted. + string response_nonce = 6; + + // This is populated when the previous :ref:`DiscoveryResponse ` + // failed to update configuration. The *message* field in *error_details* + // provides the Envoy internal exception related to the failure. + google.rpc.Status error_detail = 7; +} + +// [#next-free-field: 7] +message DeltaDiscoveryResponse { + option (udpa.annotations.versioning).previous_message_type = + "envoy.api.v2.DeltaDiscoveryResponse"; + + // The version of the response data (used for debugging). + string system_version_info = 1; + + // The response resources. These are typed resources, whose types must match + // the type_url field. + repeated Resource resources = 2; + + // field id 3 IS available! + + // Type URL for resources. Identifies the xDS API when muxing over ADS. + // Must be consistent with the type_url in the Any within 'resources' if 'resources' is non-empty. + string type_url = 4; + + // Resources names of resources that have be deleted and to be removed from the xDS Client. + // Removed resources for missing resources can be ignored. + repeated string removed_resources = 6; + + // The nonce provides a way for DeltaDiscoveryRequests to uniquely + // reference a DeltaDiscoveryResponse when (N)ACKing. The nonce is required. + string nonce = 5; +} + +message Resource { + option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Resource"; + + // The resource's name, to distinguish it from others of the same type of resource. + string name = 3; + + // The aliases are a list of other names that this resource can go by. + repeated string aliases = 4; + + // The resource level version. It allows xDS to track the state of individual + // resources. + string version = 1; + + // The resource being tracked. + google.protobuf.Any resource = 2; +} From 369efa2bbe777e52b1bbb28e13e05711cb5933bf Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Wed, 22 Jul 2020 01:03:36 +0000 Subject: [PATCH 2/9] xds: support load reporting all clusters option and fix actual report interval measurement (#7209) - Add support for send_all_clusters field in LRS response. When it is set to true, just send load reports for clusters that the client is currently tracking (aka, is sending load to). - The actual load report interval (in each ClusterStats message, which contains the stats for each cluster:eds_service) should be tracked individually. --- .../java/io/grpc/xds/LoadReportClient.java | 88 ++++++++++--------- .../io/grpc/xds/LoadReportClientTest.java | 67 ++++++++------ 2 files changed, 88 insertions(+), 67 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/LoadReportClient.java b/xds/src/main/java/io/grpc/xds/LoadReportClient.java index 525b645d610..1314583cfa7 100644 --- a/xds/src/main/java/io/grpc/xds/LoadReportClient.java +++ b/xds/src/main/java/io/grpc/xds/LoadReportClient.java @@ -67,7 +67,7 @@ final class LoadReportClient { private final BackoffPolicy.Provider backoffPolicyProvider; // Sources of load stats data for each cluster:cluster_service. - private final Map> loadStatsStoreMap = new HashMap<>(); + private final Map> loadStatsEntities = new HashMap<>(); private boolean started; @Nullable @@ -148,18 +148,18 @@ void stopLoadReporting() { void addLoadStatsStore( String clusterName, @Nullable String clusterServiceName, LoadStatsStore loadStatsStore) { checkState( - !loadStatsStoreMap.containsKey(clusterName) - || !loadStatsStoreMap.get(clusterName).containsKey(clusterServiceName), + !loadStatsEntities.containsKey(clusterName) + || !loadStatsEntities.get(clusterName).containsKey(clusterServiceName), "load stats for cluster: %s, cluster service: %s already exists", clusterName, clusterServiceName); logger.log( XdsLogLevel.INFO, "Add load stats for cluster: {0}, cluster_service: {1}", clusterName, clusterServiceName); - if (!loadStatsStoreMap.containsKey(clusterName)) { - loadStatsStoreMap.put(clusterName, new HashMap()); + if (!loadStatsEntities.containsKey(clusterName)) { + loadStatsEntities.put(clusterName, new HashMap()); } - Map clusterLoadStatsStores = loadStatsStoreMap.get(clusterName); - clusterLoadStatsStores.put(clusterServiceName, loadStatsStore); + Map clusterLoadStatsEntities = loadStatsEntities.get(clusterName); + clusterLoadStatsEntities.put(clusterServiceName, new LoadStatsEntity(loadStatsStore)); } /** @@ -167,8 +167,8 @@ void addLoadStatsStore( */ void removeLoadStatsStore(String clusterName, @Nullable String clusterServiceName) { checkState( - loadStatsStoreMap.containsKey(clusterName) - && loadStatsStoreMap.get(clusterName).containsKey(clusterServiceName), + loadStatsEntities.containsKey(clusterName) + && loadStatsEntities.get(clusterName).containsKey(clusterServiceName), "load stats for cluster: %s, cluster service: %s does not exist", clusterName, clusterServiceName); logger.log( @@ -176,10 +176,10 @@ void removeLoadStatsStore(String clusterName, @Nullable String clusterServiceNam "Remove load stats for cluster: {0}, cluster_service: {1}", clusterName, clusterServiceName); - Map clusterLoadStatsStores = loadStatsStoreMap.get(clusterName); - clusterLoadStatsStores.remove(clusterServiceName); - if (clusterLoadStatsStores.isEmpty()) { - loadStatsStoreMap.remove(clusterName); + Map clusterLoadStatsEntities = loadStatsEntities.get(clusterName); + clusterLoadStatsEntities.remove(clusterServiceName); + if (clusterLoadStatsEntities.isEmpty()) { + loadStatsEntities.remove(clusterName); } } @@ -217,10 +217,8 @@ private void startLrsRpc() { private class LrsStream implements StreamObserver { - // Cluster to report loads for asked by management server. final Set clusterNames = new HashSet<>(); final LoadReportingServiceGrpc.LoadReportingServiceStub stub; - final Stopwatch reportStopwatch; StreamObserver lrsRequestWriter; boolean initialResponseReceived; boolean closed; @@ -229,15 +227,10 @@ private class LrsStream implements StreamObserver { LrsStream(LoadReportingServiceGrpc.LoadReportingServiceStub stub, Stopwatch stopwatch) { this.stub = checkNotNull(stub, "stub"); - reportStopwatch = checkNotNull(stopwatch, "stopwatch"); } void start() { lrsRequestWriter = stub.withWaitForReady().streamLoadStats(this); - reportStopwatch.reset().start(); - - // Send an initial LRS request with empty cluster stats. Management server is able to - // infer clusters the gRPC client sending loads to. LoadStatsRequest initRequest = LoadStatsRequest.newBuilder() .setNode(node) @@ -278,19 +271,12 @@ public void run() { } private void sendLoadReport() { - long interval = reportStopwatch.elapsed(TimeUnit.NANOSECONDS); - reportStopwatch.reset().start(); LoadStatsRequest.Builder requestBuilder = LoadStatsRequest.newBuilder().setNode(node); for (String name : clusterNames) { - if (loadStatsStoreMap.containsKey(name)) { - Map clusterLoadStatsStores = loadStatsStoreMap.get(name); - for (LoadStatsStore statsStore : clusterLoadStatsStores.values()) { - ClusterStats report = - statsStore.generateLoadReport() - .toBuilder() - .setLoadReportInterval(Durations.fromNanos(interval)) - .build(); - requestBuilder.addClusterStats(report); + if (loadStatsEntities.containsKey(name)) { + Map clusterLoadStatsEntities = loadStatsEntities.get(name); + for (LoadStatsEntity entity : clusterLoadStatsEntities.values()) { + requestBuilder.addClusterStats(entity.getLoadStats()); } } } @@ -317,28 +303,27 @@ private void handleResponse(LoadStatsResponse response) { if (closed) { return; } - if (!initialResponseReceived) { logger.log(XdsLogLevel.DEBUG, "Received LRS initial response:\n{0}", response); initialResponseReceived = true; } else { logger.log(XdsLogLevel.DEBUG, "Received LRS response:\n{0}", response); } - long interval = Durations.toNanos(response.getLoadReportingInterval()); - if (interval != loadReportIntervalNano) { - logger.log(XdsLogLevel.INFO, "Update load reporting interval to {0} ns", interval); - loadReportIntervalNano = interval; - callback.onReportResponse(loadReportIntervalNano); - } - if (clusterNames.size() != response.getClustersCount() - || !clusterNames.containsAll(response.getClustersList())) { + clusterNames.clear(); + if (response.getSendAllClusters()) { + clusterNames.addAll(loadStatsEntities.keySet()); + logger.log(XdsLogLevel.INFO, "Update to report loads for all clusters"); + } else { logger.log( XdsLogLevel.INFO, "Update load reporting clusters to {0}", response.getClustersList()); - clusterNames.clear(); clusterNames.addAll(response.getClustersList()); } + long interval = Durations.toNanos(response.getLoadReportingInterval()); + logger.log(XdsLogLevel.INFO, "Update load reporting interval to {0} ns", interval); + loadReportIntervalNano = interval; scheduleNextLoadReport(); + callback.onReportResponse(loadReportIntervalNano); } private void handleStreamClosed(Status status) { @@ -401,6 +386,27 @@ private void cleanUp() { } } + private final class LoadStatsEntity { + private final LoadStatsStore loadStatsStore; + private final Stopwatch stopwatch; + + private LoadStatsEntity(LoadStatsStore loadStatsStore) { + this.loadStatsStore = loadStatsStore; + this.stopwatch = stopwatchSupplier.get().reset().start(); + } + + private ClusterStats getLoadStats() { + ClusterStats stats = + loadStatsStore.generateLoadReport() + .toBuilder() + .setLoadReportInterval( + Durations.fromNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS))) + .build(); + stopwatch.reset().start(); + return stats; + } + } + /** * Callbacks for passing information received from client load reporting responses to xDS load * balancer, such as the load reporting interval requested by the traffic director. diff --git a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java index 8c2e6f5a9fc..4b8645a7e93 100644 --- a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java +++ b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java @@ -56,7 +56,9 @@ import io.grpc.testing.GrpcCleanupRule; import io.grpc.xds.LoadReportClient.LoadReportCallback; import java.util.ArrayDeque; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -178,7 +180,7 @@ public void cancelled(Context context) { when(backoffPolicy1.nextBackoffNanos()) .thenReturn(TimeUnit.SECONDS.toNanos(1L), TimeUnit.SECONDS.toNanos(10L)); when(backoffPolicy2.nextBackoffNanos()) - .thenReturn(TimeUnit.SECONDS.toNanos(1L), TimeUnit.SECONDS.toNanos(10L)); + .thenReturn(TimeUnit.SECONDS.toNanos(2L), TimeUnit.SECONDS.toNanos(20L)); lrsClient = new LoadReportClient( logId, @@ -216,15 +218,17 @@ public void typicalWorkflow() { responseObserver.onNext(buildLrsResponse(ImmutableList.of(cluster1), 1000)); inOrder.verify(callback).onReportResponse(1000); - ArgumentMatcher expectedLoadReportMatcher = - new LoadStatsRequestMatcher(ImmutableList.of(rawStats1), 1000); + ClusterStats expectedStats1 = + rawStats1.toBuilder().setLoadReportInterval(Durations.fromNanos(1000)).build(); fakeClock.forwardNanos(999); inOrder.verifyNoMoreInteractions(); fakeClock.forwardNanos(1); - inOrder.verify(requestObserver).onNext(argThat(expectedLoadReportMatcher)); + inOrder.verify(requestObserver) + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats1)))); fakeClock.forwardNanos(1000); - inOrder.verify(requestObserver).onNext(argThat(expectedLoadReportMatcher)); + inOrder.verify(requestObserver) + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats1)))); String cluster2 = "cluster-bar.googleapis.com"; ClusterStats rawStats2 = generateClusterLoadStats(cluster2, null); @@ -236,28 +240,38 @@ public void typicalWorkflow() { responseObserver.onNext(buildLrsResponse(ImmutableList.of(cluster1), 2000)); inOrder.verify(callback).onReportResponse(2000); + expectedStats1 = + rawStats1.toBuilder().setLoadReportInterval(Durations.fromNanos(2000)).build(); fakeClock.forwardNanos(1000); inOrder.verifyNoMoreInteractions(); - fakeClock.forwardNanos(1000); inOrder.verify(requestObserver) - .onNext(argThat(new LoadStatsRequestMatcher(ImmutableList.of(rawStats1), 2000))); - - // Management server asks to report loads for cluster1 and cluster2. - responseObserver.onNext(buildLrsResponse(ImmutableList.of(cluster1, cluster2), 2000)); + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats1)))); + + // Management server asks to report loads for all clusters. + responseObserver.onNext( + LoadStatsResponse.newBuilder() + .setSendAllClusters(true) + .setLoadReportingInterval(Durations.fromNanos(2000)) + .build()); + inOrder.verify(callback).onReportResponse(2000); + ClusterStats expectedStats2 = + rawStats2.toBuilder().setLoadReportInterval(Durations.fromNanos(2000 + 2000)).build(); fakeClock.forwardNanos(2000); inOrder.verify(requestObserver) .onNext( argThat( - new LoadStatsRequestMatcher(ImmutableList.of(rawStats1, rawStats2), 2000))); + new LoadStatsRequestMatcher(Arrays.asList(expectedStats1, expectedStats2)))); // Load reports for cluster1 is no longer wanted. - responseObserver.onNext(buildLrsResponse(ImmutableList.of(cluster2), 2000)); + responseObserver.onNext(buildLrsResponse(Collections.singletonList(cluster2), 2000)); + expectedStats2 = + rawStats2.toBuilder().setLoadReportInterval(Durations.fromNanos(2000)).build(); fakeClock.forwardNanos(2000); inOrder.verify(requestObserver) - .onNext(argThat(new LoadStatsRequestMatcher(ImmutableList.of(rawStats2), 2000))); + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats2)))); // Management server asks loads for a cluster that client has no load data. responseObserver @@ -331,7 +345,7 @@ public void lrsStreamClosedAndRetried() { // Balancer sends a response asking for loads of the cluster. responseObserver - .onNext(buildLrsResponse(ImmutableList.of(clusterName), 0)); + .onNext(buildLrsResponse(ImmutableList.of(clusterName), 5)); // Then breaks the RPC responseObserver.onError(Status.UNAVAILABLE.asException()); @@ -348,12 +362,12 @@ public void lrsStreamClosedAndRetried() { fakeClock.forwardNanos(4); responseObserver.onError(Status.UNAVAILABLE.asException()); - // Will be on the first retry (1s) of backoff sequence 2. + // Will be on the first retry (2s) of backoff sequence 2. inOrder.verify(backoffPolicy2).nextBackoffNanos(); assertEquals(1, fakeClock.numPendingTasks(LRS_RPC_RETRY_TASK_FILTER)); // Fast-forward to a moment before the retry, the time spent in the last try is deducted. - fakeClock.forwardNanos(TimeUnit.SECONDS.toNanos(1) - 4 - 1); + fakeClock.forwardNanos(TimeUnit.SECONDS.toNanos(2) - 4 - 1); verifyNoMoreInteractions(mockLoadReportingService); // Then time for retry fakeClock.forwardNanos(1); @@ -368,8 +382,13 @@ public void lrsStreamClosedAndRetried() { responseObserver .onNext(buildLrsResponse(ImmutableList.of(clusterName), 10)); fakeClock.forwardNanos(10); + ClusterStats expectedStats = + stats.toBuilder() + .setLoadReportInterval( + Durations.add(Durations.fromSeconds(1 + 10 + 2), Durations.fromNanos(10))) + .build(); verify(requestObserver) - .onNext(argThat(new LoadStatsRequestMatcher(ImmutableList.of(stats), 10))); + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats)))); // Wrapping up verify(backoffPolicyProvider, times(2)).get(); @@ -466,9 +485,9 @@ private static ClusterStats generateClusterLoadStats( UpstreamLocalityStats.newBuilder() .setLocality( Locality.newBuilder() - .setRegion("region-foo") - .setZone("zone-bar") - .setSubZone("subzone-baz")) + .setRegion(clusterName + "-region-foo") + .setZone(clusterName + "-zone-bar") + .setSubZone(clusterName + "-subzone-baz")) .setTotalRequestsInProgress(callsInProgress) .setTotalSuccessfulRequests(callsSucceeded) .setTotalErrorRequests(callsFailed) @@ -491,13 +510,9 @@ private static ClusterStats generateClusterLoadStats( private static class LoadStatsRequestMatcher implements ArgumentMatcher { private final Map expectedStats = new HashMap<>(); - LoadStatsRequestMatcher(Collection clusterStats, long expectedIntervalNano) { + LoadStatsRequestMatcher(Collection clusterStats) { for (ClusterStats stats : clusterStats) { - ClusterStats statsWithInterval = - stats.toBuilder() - .setLoadReportInterval(Durations.fromNanos(expectedIntervalNano)) - .build(); - expectedStats.put(statsWithInterval.getClusterName(), statsWithInterval); + expectedStats.put(stats.getClusterName(), stats); } } From 78206c60a38bc0546902ee2cc288675a23d4c438 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Tue, 28 Jul 2020 08:56:41 -0700 Subject: [PATCH 3/9] xds: refactor usage of Node in Bootstrap to EnvoyProtoData.Node In preparation for xds-v3 support. --- .../main/java/io/grpc/xds/Bootstrapper.java | 69 +--- .../java/io/grpc/xds/EdsLoadBalancer.java | 2 +- .../main/java/io/grpc/xds/EnvoyProtoData.java | 353 +++++++++++++++++- .../java/io/grpc/xds/LoadStatsStoreImpl.java | 2 +- .../main/java/io/grpc/xds/XdsClientImpl.java | 32 +- .../xds/XdsClientWrapperForServerSds.java | 2 +- .../java/io/grpc/xds/XdsNameResolver.java | 2 +- .../sds/ClientSslContextProviderFactory.java | 2 +- .../sds/ServerSslContextProviderFactory.java | 2 +- .../java/io/grpc/xds/BootstrapperTest.java | 53 ++- .../java/io/grpc/xds/EdsLoadBalancerTest.java | 7 +- .../java/io/grpc/xds/EnvoyProtoDataTest.java | 97 ++++- .../io/grpc/xds/LoadStatsStoreImplTest.java | 2 +- .../java/io/grpc/xds/XdsClientImplTest.java | 8 +- .../xds/XdsClientImplTestForListener.java | 36 +- .../java/io/grpc/xds/XdsClientTestHelper.java | 4 +- .../xds/XdsNameResolverIntegrationTest.java | 2 +- 17 files changed, 520 insertions(+), 155 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/Bootstrapper.java b/xds/src/main/java/io/grpc/xds/Bootstrapper.java index cceb900c027..f967d43d8fd 100644 --- a/xds/src/main/java/io/grpc/xds/Bootstrapper.java +++ b/xds/src/main/java/io/grpc/xds/Bootstrapper.java @@ -17,17 +17,13 @@ package io.grpc.xds; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ListValue; -import com.google.protobuf.NullValue; -import com.google.protobuf.Struct; -import com.google.protobuf.Value; -import io.envoyproxy.envoy.api.v2.core.Locality; -import io.envoyproxy.envoy.api.v2.core.Node; import io.grpc.Internal; import io.grpc.internal.GrpcUtil; import io.grpc.internal.GrpcUtil.GrpcBuildVersion; import io.grpc.internal.JsonParser; import io.grpc.internal.JsonUtil; +import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.XdsLogger.XdsLogLevel; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -133,34 +129,24 @@ static BootstrapInfo parseConfig(String rawData) throws IOException { } Map metadata = JsonUtil.getObject(rawNode, "metadata"); if (metadata != null) { - Struct.Builder structBuilder = Struct.newBuilder(); - for (Map.Entry entry : metadata.entrySet()) { - logger.log( - XdsLogLevel.INFO, - "Node metadata field {0}: {1}", entry.getKey(), entry.getValue()); - structBuilder.putFields(entry.getKey(), convertToValue(entry.getValue())); - } - nodeBuilder.setMetadata(structBuilder); + nodeBuilder.setMetadata(metadata); } Map rawLocality = JsonUtil.getObject(rawNode, "locality"); if (rawLocality != null) { - Locality.Builder localityBuilder = Locality.newBuilder(); - if (rawLocality.containsKey("region")) { - String region = JsonUtil.getString(rawLocality, "region"); + String region = JsonUtil.getString(rawLocality, "region"); + String zone = JsonUtil.getString(rawLocality, "zone"); + String subZone = JsonUtil.getString(rawLocality, "sub_zone"); + if (region != null) { logger.log(XdsLogLevel.INFO, "Locality region: {0}", region); - localityBuilder.setRegion(region); } if (rawLocality.containsKey("zone")) { - String zone = JsonUtil.getString(rawLocality, "zone"); logger.log(XdsLogLevel.INFO, "Locality zone: {0}", zone); - localityBuilder.setZone(zone); } if (rawLocality.containsKey("sub_zone")) { - String subZone = JsonUtil.getString(rawLocality, "sub_zone"); logger.log(XdsLogLevel.INFO, "Locality sub_zone: {0}", subZone); - localityBuilder.setSubZone(subZone); } - nodeBuilder.setLocality(localityBuilder); + Locality locality = new Locality(region, zone, subZone); + nodeBuilder.setLocality(locality); } } GrpcBuildVersion buildVersion = GrpcUtil.getGrpcBuildVersion(); @@ -173,43 +159,6 @@ static BootstrapInfo parseConfig(String rawData) throws IOException { return new BootstrapInfo(servers, nodeBuilder.build()); } - /** - * Converts Java representation of the given JSON value to protobuf's {@link - * com.google.protobuf.Value} representation. - * - *

The given {@code rawObject} must be a valid JSON value in Java representation, which is - * either a {@code Map}, {@code List}, {@code String}, {@code Double}, - * {@code Boolean}, or {@code null}. - */ - private static Value convertToValue(Object rawObject) { - Value.Builder valueBuilder = Value.newBuilder(); - if (rawObject == null) { - valueBuilder.setNullValue(NullValue.NULL_VALUE); - } else if (rawObject instanceof Double) { - valueBuilder.setNumberValue((Double) rawObject); - } else if (rawObject instanceof String) { - valueBuilder.setStringValue((String) rawObject); - } else if (rawObject instanceof Boolean) { - valueBuilder.setBoolValue((Boolean) rawObject); - } else if (rawObject instanceof Map) { - Struct.Builder structBuilder = Struct.newBuilder(); - @SuppressWarnings("unchecked") - Map map = (Map) rawObject; - for (Map.Entry entry : map.entrySet()) { - structBuilder.putFields(entry.getKey(), convertToValue(entry.getValue())); - } - valueBuilder.setStructValue(structBuilder); - } else if (rawObject instanceof List) { - ListValue.Builder listBuilder = ListValue.newBuilder(); - List list = (List) rawObject; - for (Object obj : list) { - listBuilder.addValues(convertToValue(obj)); - } - valueBuilder.setListValue(listBuilder); - } - return valueBuilder.build(); - } - /** * Data class containing channel credentials configurations for xDS protocol communication. */ diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java index 34e90c7c7a1..5c5c1889fca 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java @@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.envoyproxy.envoy.api.v2.core.Node; import io.grpc.Attributes; import io.grpc.InternalLogId; import io.grpc.LoadBalancer; @@ -38,6 +37,7 @@ import io.grpc.xds.EnvoyProtoData.DropOverload; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.LocalityStore.LocalityStoreFactory; import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.EndpointWatcher; diff --git a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java index be691337f22..eeae2f96174 100644 --- a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java +++ b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java @@ -22,6 +22,10 @@ import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; +import com.google.protobuf.ListValue; +import com.google.protobuf.NullValue; +import com.google.protobuf.Struct; +import com.google.protobuf.Value; import com.google.re2j.Pattern; import com.google.re2j.PatternSyntaxException; import io.envoyproxy.envoy.type.v3.FractionalPercent; @@ -34,6 +38,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import javax.annotation.Nullable; @@ -135,17 +140,345 @@ public String toString() { } /** - * See corresponding Envoy proto message {@link io.envoyproxy.envoy.api.v2.core.Locality}. + * See corresponding Envoy proto message {@link io.envoyproxy.envoy.config.core.v3.Node}. + */ + public static final class Node { + + private final String id; + private final String cluster; + @Nullable + private final Map metadata; + @Nullable + private final Locality locality; + private final List

listeningAddresses; + private final String buildVersion; + private final String userAgentName; + @Nullable + private final String userAgentVersion; + private final List clientFeatures; + + private Node( + String id, String cluster, @Nullable Map metadata, @Nullable Locality locality, + List
listeningAddresses, String buildVersion, String userAgentName, + @Nullable String userAgentVersion, List clientFeatures) { + this.id = checkNotNull(id, "id"); + this.cluster = checkNotNull(cluster, "cluster"); + this.metadata = metadata; + this.locality = locality; + this.listeningAddresses = Collections.unmodifiableList( + checkNotNull(listeningAddresses, "listeningAddresses")); + this.buildVersion = checkNotNull(buildVersion, "buildVersion"); + this.userAgentName = checkNotNull(userAgentName, "userAgentName"); + this.userAgentVersion = userAgentVersion; + this.clientFeatures = Collections.unmodifiableList( + checkNotNull(clientFeatures, "clientFeatures")); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("id", id) + .add("cluster", cluster) + .add("metadata", metadata) + .add("locality", locality) + .add("listeningAddresses", listeningAddresses) + .add("buildVersion", buildVersion) + .add("userAgentName", userAgentName) + .add("userAgentVersion", userAgentVersion) + .add("clientFeatures", clientFeatures) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Node node = (Node) o; + return Objects.equals(id, node.id) + && Objects.equals(cluster, node.cluster) + && Objects.equals(metadata, node.metadata) + && Objects.equals(locality, node.locality) + && Objects.equals(listeningAddresses, node.listeningAddresses) + && Objects.equals(buildVersion, node.buildVersion) + && Objects.equals(userAgentName, node.userAgentName) + && Objects.equals(userAgentVersion, node.userAgentVersion) + && Objects.equals(clientFeatures, node.clientFeatures); + } + + @Override + public int hashCode() { + return Objects + .hash(id, cluster, metadata, locality, listeningAddresses, buildVersion, userAgentName, + userAgentVersion, clientFeatures); + } + + static final class Builder { + private String id = ""; + private String cluster = ""; + @Nullable + private Map metadata; + @Nullable + private Locality locality; + private final List
listeningAddresses = new ArrayList<>(); + private String buildVersion = ""; + private String userAgentName = ""; + @Nullable + private String userAgentVersion; + private final List clientFeatures = new ArrayList<>(); + + private Builder() { + } + + Builder setId(String id) { + this.id = checkNotNull(id, "id"); + return this; + } + + Builder setCluster(String cluster) { + this.cluster = checkNotNull(cluster, "cluster"); + return this; + } + + Builder setMetadata(Map metadata) { + this.metadata = checkNotNull(metadata, "metadata"); + return this; + } + + Builder setLocality(Locality locality) { + this.locality = checkNotNull(locality, "locality"); + return this; + } + + Builder addListeningAddresses(Address address) { + listeningAddresses.add(checkNotNull(address, "address")); + return this; + } + + Builder setBuildVersion(String buildVersion) { + this.buildVersion = checkNotNull(buildVersion, "buildVersion"); + return this; + } + + Builder setUserAgentName(String userAgentName) { + this.userAgentName = checkNotNull(userAgentName, "userAgentName"); + return this; + } + + Builder setUserAgentVersion(String userAgentVersion) { + this.userAgentVersion = checkNotNull(userAgentVersion, "userAgentVersion"); + return this; + } + + Builder addClientFeatures(String clientFeature) { + this.clientFeatures.add(checkNotNull(clientFeature, "clientFeature")); + return this; + } + + Node build() { + return new Node( + id, cluster, metadata, locality, listeningAddresses, buildVersion, userAgentName, + userAgentVersion, clientFeatures); + } + } + + static Builder newBuilder() { + return new Builder(); + } + + Builder toBuilder() { + Builder builder = new Builder().setId(id).setCluster(cluster); + if (metadata != null) { + builder.setMetadata(metadata); + } + if (locality != null) { + builder.setLocality(locality); + } + builder.listeningAddresses.addAll(listeningAddresses); + return builder; + } + + String getId() { + return id; + } + + String getCluster() { + return cluster; + } + + @Nullable + Map getMetadata() { + return metadata; + } + + @Nullable + Locality getLocality() { + return locality; + } + + List
getListeningAddresses() { + return listeningAddresses; + } + + io.envoyproxy.envoy.config.core.v3.Node toEnvoyProtoNode() { + io.envoyproxy.envoy.config.core.v3.Node.Builder builder = + io.envoyproxy.envoy.config.core.v3.Node.newBuilder(); + builder.setId(id); + builder.setCluster(cluster); + if (metadata != null) { + Struct.Builder structBuilder = Struct.newBuilder(); + for (Map.Entry entry : metadata.entrySet()) { + structBuilder.putFields(entry.getKey(), convertToValue(entry.getValue())); + } + builder.setMetadata(structBuilder); + } + if (locality != null) { + builder.setLocality(locality.toEnvoyProtoLocality()); + } + for (Address address : listeningAddresses) { + builder.addListeningAddresses(address.toEnvoyProtoAddress()); + } + builder.setUserAgentName(userAgentName); + if (userAgentVersion != null) { + builder.setUserAgentVersion(userAgentVersion); + } + builder.addAllClientFeatures(clientFeatures); + return builder.build(); + } + + @SuppressWarnings("deprecation") // Deprecated v2 API setBuildVersion(). + public io.envoyproxy.envoy.api.v2.core.Node toEnvoyProtoNodeV2() { + io.envoyproxy.envoy.api.v2.core.Node.Builder builder = + io.envoyproxy.envoy.api.v2.core.Node.newBuilder(); + builder.setId(id); + builder.setCluster(cluster); + if (metadata != null) { + Struct.Builder structBuilder = Struct.newBuilder(); + for (Map.Entry entry : metadata.entrySet()) { + structBuilder.putFields(entry.getKey(), convertToValue(entry.getValue())); + } + builder.setMetadata(structBuilder); + } + if (locality != null) { + builder.setLocality(locality.toEnvoyProtoLocalityV2()); + } + for (Address address : listeningAddresses) { + builder.addListeningAddresses(address.toEnvoyProtoAddressV2()); + } + builder.setBuildVersion(buildVersion); + builder.setUserAgentName(userAgentName); + if (userAgentVersion != null) { + builder.setUserAgentVersion(userAgentVersion); + } + builder.addAllClientFeatures(clientFeatures); + return builder.build(); + } + } + + /** + * Converts Java representation of the given JSON value to protobuf's {@link + * com.google.protobuf.Value} representation. + * + *

The given {@code rawObject} must be a valid JSON value in Java representation, which is + * either a {@code Map}, {@code List}, {@code String}, {@code Double}, {@code + * Boolean}, or {@code null}. + */ + private static Value convertToValue(Object rawObject) { + Value.Builder valueBuilder = Value.newBuilder(); + if (rawObject == null) { + valueBuilder.setNullValue(NullValue.NULL_VALUE); + } else if (rawObject instanceof Double) { + valueBuilder.setNumberValue((Double) rawObject); + } else if (rawObject instanceof String) { + valueBuilder.setStringValue((String) rawObject); + } else if (rawObject instanceof Boolean) { + valueBuilder.setBoolValue((Boolean) rawObject); + } else if (rawObject instanceof Map) { + Struct.Builder structBuilder = Struct.newBuilder(); + @SuppressWarnings("unchecked") + Map map = (Map) rawObject; + for (Map.Entry entry : map.entrySet()) { + structBuilder.putFields(entry.getKey(), convertToValue(entry.getValue())); + } + valueBuilder.setStructValue(structBuilder); + } else if (rawObject instanceof List) { + ListValue.Builder listBuilder = ListValue.newBuilder(); + List list = (List) rawObject; + for (Object obj : list) { + listBuilder.addValues(convertToValue(obj)); + } + valueBuilder.setListValue(listBuilder); + } + return valueBuilder.build(); + } + + /** + * See corresponding Envoy proto message {@link io.envoyproxy.envoy.config.core.v3.Address}. + */ + static final class Address { + private final String address; + private final int port; + + Address(String address, int port) { + this.address = checkNotNull(address, "address"); + this.port = port; + } + + io.envoyproxy.envoy.config.core.v3.Address toEnvoyProtoAddress() { + return + io.envoyproxy.envoy.config.core.v3.Address.newBuilder().setSocketAddress( + io.envoyproxy.envoy.config.core.v3.SocketAddress.newBuilder().setAddress(address) + .setPortValue(port)).build(); + } + + io.envoyproxy.envoy.api.v2.core.Address toEnvoyProtoAddressV2() { + return + io.envoyproxy.envoy.api.v2.core.Address.newBuilder().setSocketAddress( + io.envoyproxy.envoy.api.v2.core.SocketAddress.newBuilder().setAddress(address) + .setPortValue(port)).build(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("address", address) + .add("port", port) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Address address1 = (Address) o; + return port == address1.port && Objects.equals(address, address1.address); + } + + @Override + public int hashCode() { + return Objects.hash(address, port); + } + } + + /** + * See corresponding Envoy proto message {@link io.envoyproxy.envoy.config.core.v3.Locality}. */ static final class Locality { private final String region; private final String zone; private final String subZone; - Locality(String region, String zone, String subZone) { - this.region = region; - this.zone = zone; - this.subZone = subZone; + Locality(@Nullable String region, @Nullable String zone, @Nullable String subZone) { + this.region = region == null ? "" : region; + this.zone = zone == null ? "" : zone; + this.subZone = subZone == null ? "" : subZone; } static Locality fromEnvoyProtoLocality(io.envoyproxy.envoy.config.core.v3.Locality locality) { @@ -163,7 +496,15 @@ static Locality fromEnvoyProtoLocalityV2(io.envoyproxy.envoy.api.v2.core.Localit /* subZone = */ locality.getSubZone()); } - io.envoyproxy.envoy.api.v2.core.Locality toEnvoyProtoLocality() { + io.envoyproxy.envoy.config.core.v3.Locality toEnvoyProtoLocality() { + return io.envoyproxy.envoy.config.core.v3.Locality.newBuilder() + .setRegion(region) + .setZone(zone) + .setSubZone(subZone) + .build(); + } + + io.envoyproxy.envoy.api.v2.core.Locality toEnvoyProtoLocalityV2() { return io.envoyproxy.envoy.api.v2.core.Locality.newBuilder() .setRegion(region) .setZone(zone) diff --git a/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java b/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java index 37ff09d91be..42ba84e8508 100644 --- a/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java +++ b/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java @@ -74,7 +74,7 @@ public ClusterStats generateLoadReport() { for (Map.Entry entry : localityLoadCounters.entrySet()) { ClientLoadSnapshot snapshot = entry.getValue().snapshot(); UpstreamLocalityStats.Builder localityStatsBuilder = - UpstreamLocalityStats.newBuilder().setLocality(entry.getKey().toEnvoyProtoLocality()); + UpstreamLocalityStats.newBuilder().setLocality(entry.getKey().toEnvoyProtoLocalityV2()); localityStatsBuilder .setTotalSuccessfulRequests(snapshot.getCallsSucceeded()) .setTotalErrorRequests(snapshot.getCallsFailed()) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index e8bd49906b3..ddaec29abee 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -27,14 +27,10 @@ import com.google.protobuf.Any; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.MessageOrBuilder; -import com.google.protobuf.Struct; -import com.google.protobuf.Value; import com.google.protobuf.util.JsonFormat; import com.google.rpc.Code; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; import io.envoyproxy.envoy.api.v2.DiscoveryResponse; -import io.envoyproxy.envoy.api.v2.core.Node; -import io.envoyproxy.envoy.api.v2.core.SocketAddress; import io.envoyproxy.envoy.config.cluster.v3.Cluster; import io.envoyproxy.envoy.config.cluster.v3.Cluster.DiscoveryType; import io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig; @@ -62,6 +58,7 @@ import io.grpc.xds.EnvoyProtoData.DropOverload; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.StructOrError; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LoadReportClient.LoadReportCallback; @@ -458,18 +455,13 @@ void watchListenerData(int port, ListenerWatcher watcher) { /** In case of Listener watcher metadata to be updated to include port. */ private void updateNodeMetadataForListenerRequest(int port) { - Struct newMetadata = node.getMetadata().toBuilder() - .putFields("TRAFFICDIRECTOR_PROXYLESS", - Value.newBuilder().setStringValue("1").build()) - .build(); - io.envoyproxy.envoy.api.v2.core.Address listeningAddress = - io.envoyproxy.envoy.api.v2.core.Address.newBuilder() - .setSocketAddress( - SocketAddress.newBuilder() - .setAddress("0.0.0.0") - .setPortValue(port) - .build()) - .build(); + Map newMetadata = new HashMap<>(); + if (node.getMetadata() != null) { + newMetadata.putAll(node.getMetadata()); + } + newMetadata.put("TRAFFICDIRECTOR_PROXYLESS", "1"); + EnvoyProtoData.Address listeningAddress = + new EnvoyProtoData.Address("0.0.0.0", port); node = node.toBuilder().setMetadata(newMetadata).addListeningAddresses(listeningAddress).build(); } @@ -483,7 +475,7 @@ void reportClientStats( logId, targetName, channel, - node, + node.toEnvoyProtoNodeV2(), syncContext, timeService, backoffPolicyProvider, @@ -1480,7 +1472,7 @@ private void sendXdsRequest(String typeUrl, Collection resourceNames) { DiscoveryRequest .newBuilder() .setVersionInfo(version) - .setNode(node) + .setNode(node.toEnvoyProtoNodeV2()) .addAllResourceNames(resourceNames) .setTypeUrl(typeUrl) .setResponseNonce(nonce) @@ -1514,7 +1506,7 @@ private void sendAckRequest(String typeUrl, Collection resourceNames, DiscoveryRequest .newBuilder() .setVersionInfo(versionInfo) - .setNode(node) + .setNode(node.toEnvoyProtoNodeV2()) .addAllResourceNames(resourceNames) .setTypeUrl(typeUrl) .setResponseNonce(nonce) @@ -1561,7 +1553,7 @@ private void sendNackRequest(String typeUrl, Collection resourceNames, DiscoveryRequest .newBuilder() .setVersionInfo(versionInfo) - .setNode(node) + .setNode(node.toEnvoyProtoNodeV2()) .addAllResourceNames(resourceNames) .setTypeUrl(typeUrl) .setResponseNonce(nonce) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java b/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java index 79873e9dfa0..95e7cb9e06f 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java @@ -20,7 +20,6 @@ import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; -import io.envoyproxy.envoy.api.v2.core.Node; import io.grpc.Internal; import io.grpc.InternalLogId; import io.grpc.Status; @@ -28,6 +27,7 @@ import io.grpc.internal.ExponentialBackoffPolicy; import io.grpc.internal.GrpcUtil; import io.grpc.internal.SharedResourceHolder; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyServerProtoData.CidrRange; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; import io.grpc.xds.EnvoyServerProtoData.FilterChain; diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 47946949b49..e0c03e5f907 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -26,7 +26,6 @@ import com.google.common.collect.Iterables; import com.google.gson.Gson; import com.google.re2j.Pattern; -import io.envoyproxy.envoy.api.v2.core.Node; import io.grpc.Attributes; import io.grpc.EquivalentAddressGroup; import io.grpc.InternalLogId; @@ -39,6 +38,7 @@ import io.grpc.xds.Bootstrapper.BootstrapInfo; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.ClusterWeight; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.Route; import io.grpc.xds.EnvoyProtoData.RouteAction; import io.grpc.xds.RouteMatch.FractionMatcher; diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java b/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java index bb6a636e969..84b2f8284aa 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java @@ -43,7 +43,7 @@ public SslContextProvider create(UpstreamTlsContext upstreamTlsContext) { try { return SdsClientSslContextProvider.getProvider( upstreamTlsContext, - Bootstrapper.getInstance().readBootstrap().getNode(), + Bootstrapper.getInstance().readBootstrap().getNode().toEnvoyProtoNodeV2(), Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setNameFormat("client-sds-sslcontext-provider-%d") .setDaemon(true) diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java b/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java index fb271a32dc7..1cd2cfa8e9b 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java @@ -45,7 +45,7 @@ public SslContextProvider create( try { return SdsServerSslContextProvider.getProvider( downstreamTlsContext, - Bootstrapper.getInstance().readBootstrap().getNode(), + Bootstrapper.getInstance().readBootstrap().getNode().toEnvoyProtoNodeV2(), Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setNameFormat("server-sds-sslcontext-provider-%d") .setDaemon(true) diff --git a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java index 37ef4161259..10d2c109167 100644 --- a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java @@ -18,15 +18,14 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import com.google.protobuf.Struct; -import com.google.protobuf.Value; -import io.envoyproxy.envoy.api.v2.core.Locality; -import io.envoyproxy.envoy.api.v2.core.Node; import io.grpc.internal.GrpcUtil; import io.grpc.internal.GrpcUtil.GrpcBuildVersion; import io.grpc.xds.Bootstrapper.BootstrapInfo; import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.Node; import java.io.IOException; import java.util.List; import org.junit.Rule; @@ -82,16 +81,13 @@ public void parseBootstrap_validData_singleXdsServer() throws IOException { getNodeBuilder() .setId("ENVOY_NODE_ID") .setCluster("ENVOY_CLUSTER") - .setLocality( - Locality.newBuilder() - .setRegion("ENVOY_REGION").setZone("ENVOY_ZONE").setSubZone("ENVOY_SUBZONE")) + .setLocality(new Locality("ENVOY_REGION", "ENVOY_ZONE", "ENVOY_SUBZONE")) .setMetadata( - Struct.newBuilder() - .putFields("TRAFFICDIRECTOR_INTERCEPTION_PORT", - Value.newBuilder().setStringValue("ENVOY_PORT").build()) - .putFields("TRAFFICDIRECTOR_NETWORK_NAME", - Value.newBuilder().setStringValue("VPC_NETWORK_NAME").build()) - .build()) + ImmutableMap.of( + "TRAFFICDIRECTOR_INTERCEPTION_PORT", + "ENVOY_PORT", + "TRAFFICDIRECTOR_NETWORK_NAME", + "VPC_NETWORK_NAME")) .build()); } @@ -145,16 +141,13 @@ public void parseBootstrap_validData_multipleXdsServers() throws IOException { getNodeBuilder() .setId("ENVOY_NODE_ID") .setCluster("ENVOY_CLUSTER") - .setLocality( - Locality.newBuilder() - .setRegion("ENVOY_REGION").setZone("ENVOY_ZONE").setSubZone("ENVOY_SUBZONE")) + .setLocality(new Locality("ENVOY_REGION", "ENVOY_ZONE", "ENVOY_SUBZONE")) .setMetadata( - Struct.newBuilder() - .putFields("TRAFFICDIRECTOR_INTERCEPTION_PORT", - Value.newBuilder().setStringValue("ENVOY_PORT").build()) - .putFields("TRAFFICDIRECTOR_NETWORK_NAME", - Value.newBuilder().setStringValue("VPC_NETWORK_NAME").build()) - .build()) + ImmutableMap.of( + "TRAFFICDIRECTOR_INTERCEPTION_PORT", + "ENVOY_PORT", + "TRAFFICDIRECTOR_NETWORK_NAME", + "VPC_NETWORK_NAME")) .build()); } @@ -201,16 +194,13 @@ public void parseBootstrap_IgnoreIrrelevantFields() throws IOException { getNodeBuilder() .setId("ENVOY_NODE_ID") .setCluster("ENVOY_CLUSTER") - .setLocality( - Locality.newBuilder() - .setRegion("ENVOY_REGION").setZone("ENVOY_ZONE").setSubZone("ENVOY_SUBZONE")) + .setLocality(new Locality("ENVOY_REGION", "ENVOY_ZONE", "ENVOY_SUBZONE")) .setMetadata( - Struct.newBuilder() - .putFields("TRAFFICDIRECTOR_INTERCEPTION_PORT", - Value.newBuilder().setStringValue("ENVOY_PORT").build()) - .putFields("TRAFFICDIRECTOR_NETWORK_NAME", - Value.newBuilder().setStringValue("VPC_NETWORK_NAME").build()) - .build()) + ImmutableMap.of( + "TRAFFICDIRECTOR_INTERCEPTION_PORT", + "ENVOY_PORT", + "TRAFFICDIRECTOR_NETWORK_NAME", + "VPC_NETWORK_NAME")) .build()); } @@ -304,7 +294,6 @@ public void parseBootstrap_serverWithoutServerUri() throws IOException { Bootstrapper.parseConfig(rawData); } - @SuppressWarnings("deprecation") private static Node.Builder getNodeBuilder() { GrpcBuildVersion buildVersion = GrpcUtil.getGrpcBuildVersion(); return diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java index ff9101b5a60..66ce45c3ff4 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java @@ -43,7 +43,6 @@ import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment.Policy.DropOverload; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; import io.envoyproxy.envoy.api.v2.DiscoveryResponse; -import io.envoyproxy.envoy.api.v2.core.Node; import io.envoyproxy.envoy.api.v2.endpoint.LbEndpoint; import io.envoyproxy.envoy.api.v2.endpoint.LocalityLbEndpoints; import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; @@ -76,6 +75,7 @@ import io.grpc.xds.Bootstrapper.ChannelCreds; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.LocalityStore.LocalityStoreFactory; import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.XdsChannelFactory; @@ -231,7 +231,8 @@ public StreamObserver streamAggregatedResources( final List serverList = ImmutableList.of( new ServerInfo("trafficdirector.googleapis.com", ImmutableList.of())); - BootstrapInfo bootstrapInfo = new BootstrapInfo(serverList, Node.getDefaultInstance()); + Node node = Node.newBuilder().build(); + BootstrapInfo bootstrapInfo = new BootstrapInfo(serverList, node); doReturn(bootstrapInfo).when(bootstrapper).readBootstrap(); if (isFullFlow) { @@ -240,7 +241,7 @@ public StreamObserver streamAggregatedResources( SERVICE_AUTHORITY, serverList, channelFactory, - Node.getDefaultInstance(), + node, syncContext, fakeClock.getScheduledExecutorService(), mock(BackoffPolicy.Provider.class), diff --git a/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java b/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java index 7e633d55bd5..fa278fb8147 100644 --- a/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java +++ b/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java @@ -18,9 +18,12 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.common.collect.ImmutableMap; import com.google.common.testing.EqualsTester; import com.google.protobuf.BoolValue; +import com.google.protobuf.Struct; import com.google.protobuf.UInt32Value; +import com.google.protobuf.Value; import com.google.re2j.Pattern; import io.envoyproxy.envoy.config.core.v3.RuntimeFractionalPercent; import io.envoyproxy.envoy.config.route.v3.QueryParameterMatcher; @@ -29,8 +32,10 @@ import io.envoyproxy.envoy.type.matcher.v3.RegexMatcher; import io.envoyproxy.envoy.type.v3.FractionalPercent; import io.envoyproxy.envoy.type.v3.Int64Range; +import io.grpc.xds.EnvoyProtoData.Address; import io.grpc.xds.EnvoyProtoData.ClusterWeight; import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.Route; import io.grpc.xds.EnvoyProtoData.RouteAction; import io.grpc.xds.EnvoyProtoData.StructOrError; @@ -63,7 +68,8 @@ public void locality_convertToAndFromLocalityProto() { assertThat(xdsLocality.getZone()).isEqualTo("test_zone"); assertThat(xdsLocality.getSubZone()).isEqualTo("test_subzone"); - io.envoyproxy.envoy.api.v2.core.Locality convertedLocality = xdsLocality.toEnvoyProtoLocality(); + io.envoyproxy.envoy.api.v2.core.Locality convertedLocality = + xdsLocality.toEnvoyProtoLocalityV2(); assertThat(convertedLocality.getRegion()).isEqualTo("test_region"); assertThat(convertedLocality.getZone()).isEqualTo("test_zone"); assertThat(convertedLocality.getSubZone()).isEqualTo("test_subzone"); @@ -84,6 +90,95 @@ public void locality_equal() { .testEquals(); } + @Test + public void convertNode() { + Node node = Node.newBuilder() + .setId("node-id") + .setCluster("cluster") + .setMetadata( + ImmutableMap.of( + "TRAFFICDIRECTOR_INTERCEPTION_PORT", + "ENVOY_PORT", + "TRAFFICDIRECTOR_NETWORK_NAME", + "VPC_NETWORK_NAME")) + .setLocality(new Locality("region", "zone", "subzone")) + .addListeningAddresses(new Address("www.foo.com", 8080)) + .addListeningAddresses(new Address("www.bar.com", 8088)) + .setBuildVersion("v1") + .setUserAgentName("agent") + .setUserAgentVersion("1.1") + .addClientFeatures("feature-1") + .addClientFeatures("feature-2") + .build(); + io.envoyproxy.envoy.config.core.v3.Node nodeProto = + io.envoyproxy.envoy.config.core.v3.Node.newBuilder() + .setId("node-id") + .setCluster("cluster") + .setMetadata(Struct.newBuilder() + .putFields("TRAFFICDIRECTOR_INTERCEPTION_PORT", + Value.newBuilder().setStringValue("ENVOY_PORT").build()) + .putFields("TRAFFICDIRECTOR_NETWORK_NAME", + Value.newBuilder().setStringValue("VPC_NETWORK_NAME").build())) + .setLocality( + io.envoyproxy.envoy.config.core.v3.Locality.newBuilder() + .setRegion("region") + .setZone("zone") + .setSubZone("subzone")) + .addListeningAddresses( + io.envoyproxy.envoy.config.core.v3.Address.newBuilder() + .setSocketAddress( + io.envoyproxy.envoy.config.core.v3.SocketAddress.newBuilder() + .setAddress("www.foo.com") + .setPortValue(8080))) + .addListeningAddresses( + io.envoyproxy.envoy.config.core.v3.Address.newBuilder() + .setSocketAddress( + io.envoyproxy.envoy.config.core.v3.SocketAddress.newBuilder() + .setAddress("www.bar.com") + .setPortValue(8088))) + .setUserAgentName("agent") + .setUserAgentVersion("1.1") + .addClientFeatures("feature-1") + .addClientFeatures("feature-2") + .build(); + assertThat(node.toEnvoyProtoNode()).isEqualTo(nodeProto); + + @SuppressWarnings("deprecation") // Deprecated v2 API setBuildVersion(). + io.envoyproxy.envoy.api.v2.core.Node nodeProtoV2 = + io.envoyproxy.envoy.api.v2.core.Node.newBuilder() + .setId("node-id") + .setCluster("cluster") + .setMetadata(Struct.newBuilder() + .putFields("TRAFFICDIRECTOR_INTERCEPTION_PORT", + Value.newBuilder().setStringValue("ENVOY_PORT").build()) + .putFields("TRAFFICDIRECTOR_NETWORK_NAME", + Value.newBuilder().setStringValue("VPC_NETWORK_NAME").build())) + .setLocality( + io.envoyproxy.envoy.api.v2.core.Locality.newBuilder() + .setRegion("region") + .setZone("zone") + .setSubZone("subzone")) + .addListeningAddresses( + io.envoyproxy.envoy.api.v2.core.Address.newBuilder() + .setSocketAddress( + io.envoyproxy.envoy.api.v2.core.SocketAddress.newBuilder() + .setAddress("www.foo.com") + .setPortValue(8080))) + .addListeningAddresses( + io.envoyproxy.envoy.api.v2.core.Address.newBuilder() + .setSocketAddress( + io.envoyproxy.envoy.api.v2.core.SocketAddress.newBuilder() + .setAddress("www.bar.com") + .setPortValue(8088))) + .setBuildVersion("v1") + .setUserAgentName("agent") + .setUserAgentVersion("1.1") + .addClientFeatures("feature-1") + .addClientFeatures("feature-2") + .build(); + assertThat(node.toEnvoyProtoNodeV2()).isEqualTo(nodeProtoV2); + } + @Test public void locality_hash() { assertThat(new Locality("region", "zone", "subzone").hashCode()) diff --git a/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java b/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java index d58fd4bb92e..96b96b67684 100644 --- a/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java +++ b/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java @@ -83,7 +83,7 @@ private static UpstreamLocalityStats buildUpstreamLocalityStats( @Nullable List metrics) { UpstreamLocalityStats.Builder builder = UpstreamLocalityStats.newBuilder() - .setLocality(locality.toEnvoyProtoLocality()) + .setLocality(locality.toEnvoyProtoLocalityV2()) .setTotalSuccessfulRequests(callsSucceed) .setTotalErrorRequests(callsFailed) .setTotalRequestsInProgress(callsInProgress) diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index 8e8002ecc0b..a63b5401c03 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -57,7 +57,6 @@ import io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource; import io.envoyproxy.envoy.api.v2.core.ConfigSource; import io.envoyproxy.envoy.api.v2.core.HealthStatus; -import io.envoyproxy.envoy.api.v2.core.Node; import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; import io.envoyproxy.envoy.api.v2.route.RedirectAction; import io.envoyproxy.envoy.api.v2.route.WeightedCluster; @@ -94,6 +93,7 @@ import io.grpc.xds.EnvoyProtoData.LbEndpoint; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.XdsClient.ClusterUpdate; import io.grpc.xds.XdsClient.ClusterWatcher; import io.grpc.xds.XdsClient.ConfigUpdate; @@ -132,7 +132,7 @@ public class XdsClientImplTest { private static final String TARGET_AUTHORITY = "foo.googleapis.com:8080"; - private static final Node NODE = Node.getDefaultInstance(); + private static final Node NODE = Node.newBuilder().build(); private static final FakeClock.TaskFilter RPC_RETRY_TASK_FILTER = new FakeClock.TaskFilter() { @Override @@ -294,7 +294,7 @@ ManagedChannel createChannel(List servers) { TARGET_AUTHORITY, servers, channelFactory, - NODE, + EnvoyProtoData.Node.newBuilder().build(), syncContext, fakeClock.getScheduledExecutorService(), backoffPolicyProvider, @@ -3732,7 +3732,7 @@ public boolean matches(DiscoveryRequest argument) { if (!resourceNames.equals(new HashSet<>(argument.getResourceNamesList()))) { return false; } - return NODE.equals(argument.getNode()); + return argument.getNode().equals(NODE.toEnvoyProtoNodeV2()); } } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java index fed260b7b11..91d22c69ee0 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java @@ -37,16 +37,12 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.Any; import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Struct; import com.google.protobuf.UInt32Value; -import com.google.protobuf.Value; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; import io.envoyproxy.envoy.api.v2.DiscoveryResponse; import io.envoyproxy.envoy.api.v2.Listener; import io.envoyproxy.envoy.api.v2.auth.DownstreamTlsContext; -import io.envoyproxy.envoy.api.v2.core.Address; import io.envoyproxy.envoy.api.v2.core.CidrRange; -import io.envoyproxy.envoy.api.v2.core.Node; import io.envoyproxy.envoy.api.v2.core.SocketAddress; import io.envoyproxy.envoy.api.v2.core.TransportSocket; import io.envoyproxy.envoy.api.v2.listener.Filter; @@ -70,6 +66,8 @@ import io.grpc.testing.GrpcCleanupRule; import io.grpc.xds.Bootstrapper.ChannelCreds; import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.EnvoyProtoData.Address; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.XdsClient.ConfigWatcher; import io.grpc.xds.XdsClient.ListenerUpdate; import io.grpc.xds.XdsClient.ListenerWatcher; @@ -78,7 +76,9 @@ import java.io.IOException; import java.util.ArrayDeque; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -109,7 +109,7 @@ public class XdsClientImplTestForListener { "type.googleapis.com/" + "envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager"; - private static final Node NODE = Node.getDefaultInstance(); + private static final Node NODE = Node.newBuilder().build(); private static final FakeClock.TaskFilter RPC_RETRY_TASK_FILTER = new FakeClock.TaskFilter() { @Override @@ -230,15 +230,12 @@ public void tearDown() { } private static Node getNodeToVerify() { - Struct newMetadata = NODE.getMetadata().toBuilder() - .putFields("TRAFFICDIRECTOR_PROXYLESS", - Value.newBuilder().setStringValue("1").build()) - .build(); - Address listeningAddress = - Address.newBuilder() - .setSocketAddress( - SocketAddress.newBuilder().setAddress("0.0.0.0").setPortValue(PORT).build()) - .build(); + Map newMetadata = new HashMap<>(); + if (NODE.getMetadata() != null) { + newMetadata.putAll(NODE.getMetadata()); + } + newMetadata.put("TRAFFICDIRECTOR_PROXYLESS", "1"); + Address listeningAddress = new Address("0.0.0.0", PORT); return NODE.toBuilder() .setMetadata(newMetadata) .addListeningAddresses(listeningAddress) @@ -249,7 +246,7 @@ private static DiscoveryRequest buildDiscoveryRequest( Node node, String versionInfo, String typeUrl, String nonce) { return DiscoveryRequest.newBuilder() .setVersionInfo(versionInfo) - .setNode(node) + .setNode(node.toEnvoyProtoNodeV2()) .setTypeUrl(typeUrl) .setResponseNonce(nonce) .build(); @@ -815,10 +812,11 @@ public void streamClosedAndRetry() { static Listener buildListenerWithFilterChain(String name, int portValue, String address, FilterChain... filterChains) { - Address listenerAddress = Address.newBuilder() - .setSocketAddress(SocketAddress.newBuilder() - .setPortValue(portValue).setAddress(address)) - .build(); + io.envoyproxy.envoy.api.v2.core.Address listenerAddress = + io.envoyproxy.envoy.api.v2.core.Address.newBuilder() + .setSocketAddress( + SocketAddress.newBuilder().setPortValue(portValue).setAddress(address)) + .build(); return Listener.newBuilder() .setName(name) diff --git a/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java index 868fb426379..9918a9073e1 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java @@ -39,7 +39,6 @@ import io.envoyproxy.envoy.api.v2.core.GrpcService; import io.envoyproxy.envoy.api.v2.core.GrpcService.GoogleGrpc; import io.envoyproxy.envoy.api.v2.core.HealthStatus; -import io.envoyproxy.envoy.api.v2.core.Node; import io.envoyproxy.envoy.api.v2.core.SelfConfigSource; import io.envoyproxy.envoy.api.v2.core.SocketAddress; import io.envoyproxy.envoy.api.v2.core.TransportSocket; @@ -51,6 +50,7 @@ import io.envoyproxy.envoy.config.listener.v2.ApiListener; import io.envoyproxy.envoy.type.FractionalPercent; import io.envoyproxy.envoy.type.FractionalPercent.DenominatorType; +import io.grpc.xds.EnvoyProtoData.Node; import java.util.List; import javax.annotation.Nullable; @@ -79,7 +79,7 @@ static DiscoveryRequest buildDiscoveryRequest(Node node, String versionInfo, return DiscoveryRequest.newBuilder() .setVersionInfo(versionInfo) - .setNode(node) + .setNode(node.toEnvoyProtoNodeV2()) .setTypeUrl(typeUrl) .addAllResourceNames(resourceNames) .setResponseNonce(nonce) diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java index fdacff15e98..4584bcc6ea7 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java @@ -37,7 +37,6 @@ import io.envoyproxy.envoy.api.v2.DiscoveryResponse; import io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource; import io.envoyproxy.envoy.api.v2.core.ConfigSource; -import io.envoyproxy.envoy.api.v2.core.Node; import io.envoyproxy.envoy.api.v2.route.Route; import io.envoyproxy.envoy.api.v2.route.RouteAction; import io.envoyproxy.envoy.api.v2.route.RouteMatch; @@ -65,6 +64,7 @@ import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.XdsClient.XdsChannelFactory; import java.io.IOException; import java.util.ArrayDeque; From 897da509b1c848024b85eb4b3dfbf4429cb32987 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Thu, 30 Jul 2020 13:41:34 -0700 Subject: [PATCH 4/9] xds: refactor AdsStream to envoy-proto-and-version-agnostic AbstractAdsStream In preparation of xds-v3 support. --- .../main/java/io/grpc/xds/XdsClientImpl.java | 699 ++++++++++++------ 1 file changed, 481 insertions(+), 218 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index ddaec29abee..4991cdf1d83 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -29,8 +29,6 @@ import com.google.protobuf.MessageOrBuilder; import com.google.protobuf.util.JsonFormat; import com.google.rpc.Code; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; import io.envoyproxy.envoy.config.cluster.v3.Cluster; import io.envoyproxy.envoy.config.cluster.v3.Cluster.DiscoveryType; import io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig; @@ -46,7 +44,9 @@ import io.envoyproxy.envoy.config.route.v3.VirtualHost; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; -import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; import io.grpc.InternalLogId; import io.grpc.ManagedChannel; import io.grpc.Status; @@ -167,7 +167,7 @@ final class XdsClientImpl extends XdsClient { private ScheduledHandle rdsRespTimer; @Nullable - private AdsStream adsStream; + private AbstractAdsStream adsStream; @Nullable private BackoffPolicy retryBackoffPolicy; @Nullable @@ -270,7 +270,7 @@ void watchConfigData(String targetAuthority, ConfigWatcher watcher) { if (adsStream == null) { startRpcStream(); } - adsStream.sendXdsRequest(ADS_TYPE_URL_LDS_V2, ImmutableList.of(ldsResourceName)); + adsStream.sendXdsRequest(ResourceType.LDS, ImmutableList.of(ldsResourceName)); ldsRespTimer = syncContext .schedule( @@ -311,7 +311,7 @@ void watchClusterData(String clusterName, ClusterWatcher watcher) { if (adsStream == null) { startRpcStream(); } - adsStream.sendXdsRequest(ADS_TYPE_URL_CDS_V2, clusterWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.CDS, clusterWatchers.keySet()); ScheduledHandle timeoutHandle = syncContext .schedule( @@ -348,7 +348,7 @@ void cancelClusterDataWatch(String clusterName, ClusterWatcher watcher) { } checkState(adsStream != null, "Severe bug: ADS stream was not created while an endpoint watcher was registered"); - adsStream.sendXdsRequest(ADS_TYPE_URL_CDS_V2, clusterWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.CDS, clusterWatchers.keySet()); } } @@ -389,7 +389,7 @@ void watchEndpointData(String clusterName, EndpointWatcher watcher) { if (adsStream == null) { startRpcStream(); } - adsStream.sendXdsRequest(ADS_TYPE_URL_EDS_V2, endpointWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.EDS, endpointWatchers.keySet()); ScheduledHandle timeoutHandle = syncContext .schedule( @@ -424,7 +424,7 @@ void cancelEndpointDataWatch(String clusterName, EndpointWatcher watcher) { // Currently in retry backoff. return; } - adsStream.sendXdsRequest(ADS_TYPE_URL_EDS_V2, endpointWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.EDS, endpointWatchers.keySet()); } } @@ -445,7 +445,7 @@ void watchListenerData(int port, ListenerWatcher watcher) { startRpcStream(); } updateNodeMetadataForListenerRequest(port); - adsStream.sendXdsRequest(ADS_TYPE_URL_LDS_V2, ImmutableList.of()); + adsStream.sendXdsRequest(ResourceType.LDS, ImmutableList.of()); ldsRespTimer = syncContext .schedule( @@ -515,9 +515,12 @@ public String toString() { */ private void startRpcStream() { checkState(adsStream == null, "Previous adsStream has not been cleared yet"); - AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub stub = - AggregatedDiscoveryServiceGrpc.newStub(channel); - adsStream = new AdsStream(stub); + io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc + .AggregatedDiscoveryServiceStub + stub = + io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.newStub( + channel); + adsStream = new AdsStreamV2(stub); adsStream.start(); logger.log(XdsLogLevel.INFO, "ADS stream started"); adsStreamRetryStopwatch.reset().start(); @@ -527,13 +530,9 @@ private void startRpcStream() { * Calls handleLdsResponseForListener or handleLdsResponseForConfigUpdate based on which watcher * was set. */ - private void handleLdsResponse(DiscoveryResponse ldsResponse) { + private void handleLdsResponse(DiscoveryResponseData ldsResponse) { checkState((configWatcher != null) != (listenerWatcher != null), "No LDS request was ever sent. Management server is doing something wrong"); - if (logger.isLoggable(XdsLogLevel.DEBUG)) { - logger.log( - XdsLogLevel.DEBUG, "Received LDS response:\n{0}", respPrinter.print(ldsResponse)); - } if (listenerWatcher != null) { handleLdsResponseForListener(ldsResponse); } else { @@ -549,13 +548,13 @@ private void handleLdsResponse(DiscoveryResponse ldsResponse) { * resolution. The response is NACKed if contains invalid data for gRPC's usage. Otherwise, an * ACK request is sent to management server. */ - private void handleLdsResponseForConfigUpdate(DiscoveryResponse ldsResponse) { + private void handleLdsResponseForConfigUpdate(DiscoveryResponseData ldsResponse) { checkState(ldsResourceName != null && configWatcher != null, "LDS request for ConfigWatcher was never sent!"); // Unpack Listener messages. - List listeners = new ArrayList<>(ldsResponse.getResourcesCount()); - List listenerNames = new ArrayList<>(ldsResponse.getResourcesCount()); + List listeners = new ArrayList<>(ldsResponse.getResourcesList().size()); + List listenerNames = new ArrayList<>(ldsResponse.getResourcesList().size()); try { for (com.google.protobuf.Any res : ldsResponse.getResourcesList()) { if (res.getTypeUrl().equals(ADS_TYPE_URL_LDS_V2)) { @@ -568,7 +567,7 @@ private void handleLdsResponseForConfigUpdate(DiscoveryResponse ldsResponse) { } catch (InvalidProtocolBufferException e) { logger.log(XdsLogLevel.WARNING, "Failed to unpack Listeners in LDS response {0}", e); adsStream.sendNackRequest( - ADS_TYPE_URL_LDS_V2, ImmutableList.of(ldsResourceName), + ResourceType.LDS, ImmutableList.of(ldsResourceName), ldsResponse.getVersionInfo(), "Malformed LDS response: " + e); return; } @@ -593,7 +592,7 @@ private void handleLdsResponseForConfigUpdate(DiscoveryResponse ldsResponse) { XdsLogLevel.WARNING, "Failed to unpack HttpConnectionManagers in Listeners of LDS response {0}", e); adsStream.sendNackRequest( - ADS_TYPE_URL_LDS_V2, ImmutableList.of(ldsResourceName), + ResourceType.LDS, ImmutableList.of(ldsResourceName), ldsResponse.getVersionInfo(), "Malformed LDS response: " + e); return; } @@ -639,11 +638,11 @@ private void handleLdsResponseForConfigUpdate(DiscoveryResponse ldsResponse) { if (errorMessage != null) { adsStream.sendNackRequest( - ADS_TYPE_URL_LDS_V2, ImmutableList.of(ldsResourceName), + ResourceType.LDS, ImmutableList.of(ldsResourceName), ldsResponse.getVersionInfo(), errorMessage); return; } - adsStream.sendAckRequest(ADS_TYPE_URL_LDS_V2, ImmutableList.of(ldsResourceName), + adsStream.sendAckRequest(ResourceType.LDS, ImmutableList.of(ldsResourceName), ldsResponse.getVersionInfo()); if (routes != null || rdsRouteConfigName != null) { @@ -665,7 +664,7 @@ private void handleLdsResponseForConfigUpdate(DiscoveryResponse ldsResponse) { logger.log( XdsLogLevel.INFO, "Use RDS to dynamically resolve route config, resource name: {0}", rdsRouteConfigName); - adsStream.sendXdsRequest(ADS_TYPE_URL_RDS_V2, ImmutableList.of(rdsRouteConfigName)); + adsStream.sendXdsRequest(ResourceType.RDS, ImmutableList.of(rdsRouteConfigName)); // Cancel the timer for fetching the previous RDS resource. if (rdsRespTimer != null) { rdsRespTimer.cancel(); @@ -684,13 +683,13 @@ private void handleLdsResponseForConfigUpdate(DiscoveryResponse ldsResponse) { } } - private void handleLdsResponseForListener(DiscoveryResponse ldsResponse) { + private void handleLdsResponseForListener(DiscoveryResponseData ldsResponse) { checkState(ldsResourceName == null && listenerPort > 0 && listenerWatcher != null, "LDS request for ListenerWatcher was never sent!"); // Unpack Listener messages. Listener requestedListener = null; - logger.log(XdsLogLevel.DEBUG, "Listener count: {0}", ldsResponse.getResourcesCount()); + logger.log(XdsLogLevel.DEBUG, "Listener count: {0}", ldsResponse.getResourcesList().size()); try { for (com.google.protobuf.Any res : ldsResponse.getResourcesList()) { if (res.getTypeUrl().equals(ADS_TYPE_URL_LDS_V2)) { @@ -706,7 +705,7 @@ private void handleLdsResponseForListener(DiscoveryResponse ldsResponse) { } catch (InvalidProtocolBufferException e) { logger.log(XdsLogLevel.WARNING, "Failed to unpack Listeners in LDS response {0}", e); adsStream.sendNackRequest( - ADS_TYPE_URL_LDS_V2, ImmutableList.of(), + ResourceType.LDS, ImmutableList.of(), ldsResponse.getVersionInfo(), "Malformed LDS response: " + e); return; } @@ -723,7 +722,7 @@ private void handleLdsResponseForListener(DiscoveryResponse ldsResponse) { } catch (InvalidProtocolBufferException e) { logger.log(XdsLogLevel.WARNING, "Failed to unpack Listener in LDS response {0}", e); adsStream.sendNackRequest( - ADS_TYPE_URL_LDS_V2, ImmutableList.of(), + ResourceType.LDS, ImmutableList.of(), ldsResponse.getVersionInfo(), "Malformed LDS response: " + e); return; } @@ -732,7 +731,7 @@ private void handleLdsResponseForListener(DiscoveryResponse ldsResponse) { listenerWatcher.onResourceDoesNotExist(":" + listenerPort); } } - adsStream.sendAckRequest(ADS_TYPE_URL_LDS_V2, ImmutableList.of(), + adsStream.sendAckRequest(ResourceType.LDS, ImmutableList.of(), ldsResponse.getVersionInfo()); if (listenerUpdate != null) { listenerWatcher.onListenerChanged(listenerUpdate); @@ -769,15 +768,12 @@ private boolean hasMatchingFilter(List filterChainsList) { * for the "xds:" URI (with the port, if any, stripped off). The response is NACKed if contains * invalid data for gRPC's usage. Otherwise, an ACK request is sent to management server. */ - private void handleRdsResponse(DiscoveryResponse rdsResponse) { - if (logger.isLoggable(XdsLogLevel.DEBUG)) { - logger.log(XdsLogLevel.DEBUG, "Received RDS response:\n{0}", respPrinter.print(rdsResponse)); - } + private void handleRdsResponse(DiscoveryResponseData rdsResponse) { checkState(adsStream.rdsResourceName != null, "Never requested for RDS resources, management server is doing something wrong"); // Unpack RouteConfiguration messages. - List routeConfigNames = new ArrayList<>(rdsResponse.getResourcesCount()); + List routeConfigNames = new ArrayList<>(rdsResponse.getResourcesList().size()); RouteConfiguration requestedRouteConfig = null; try { for (com.google.protobuf.Any res : rdsResponse.getResourcesList()) { @@ -794,7 +790,7 @@ private void handleRdsResponse(DiscoveryResponse rdsResponse) { logger.log( XdsLogLevel.WARNING, "Failed to unpack RouteConfiguration in RDS response {0}", e); adsStream.sendNackRequest( - ADS_TYPE_URL_RDS_V2, ImmutableList.of(adsStream.rdsResourceName), + ResourceType.RDS, ImmutableList.of(adsStream.rdsResourceName), rdsResponse.getVersionInfo(), "Malformed RDS response: " + e); return; } @@ -809,7 +805,7 @@ private void handleRdsResponse(DiscoveryResponse rdsResponse) { } catch (InvalidProtoDataException e) { String errorDetail = e.getMessage(); adsStream.sendNackRequest( - ADS_TYPE_URL_RDS_V2, ImmutableList.of(adsStream.rdsResourceName), + ResourceType.RDS, ImmutableList.of(adsStream.rdsResourceName), rdsResponse.getVersionInfo(), "RouteConfiguration " + requestedRouteConfig.getName() + ": cannot find a " + "valid cluster name in any virtual hosts with domains matching: " @@ -819,7 +815,7 @@ private void handleRdsResponse(DiscoveryResponse rdsResponse) { } } - adsStream.sendAckRequest(ADS_TYPE_URL_RDS_V2, ImmutableList.of(adsStream.rdsResourceName), + adsStream.sendAckRequest(ResourceType.RDS, ImmutableList.of(adsStream.rdsResourceName), rdsResponse.getVersionInfo()); // Notify the ConfigWatcher if this RDS response contains the most recently requested @@ -929,15 +925,12 @@ static VirtualHost findVirtualHostForHostName( * Response data for requested clusters is cached locally, in case of new cluster watchers * interested in the same clusters are added later. */ - private void handleCdsResponse(DiscoveryResponse cdsResponse) { - if (logger.isLoggable(XdsLogLevel.DEBUG)) { - logger.log(XdsLogLevel.DEBUG, "Received CDS response:\n{0}", respPrinter.print(cdsResponse)); - } + private void handleCdsResponse(DiscoveryResponseData cdsResponse) { adsStream.cdsRespNonce = cdsResponse.getNonce(); // Unpack Cluster messages. - List clusters = new ArrayList<>(cdsResponse.getResourcesCount()); - List clusterNames = new ArrayList<>(cdsResponse.getResourcesCount()); + List clusters = new ArrayList<>(cdsResponse.getResourcesList().size()); + List clusterNames = new ArrayList<>(cdsResponse.getResourcesList().size()); try { for (com.google.protobuf.Any res : cdsResponse.getResourcesList()) { if (res.getTypeUrl().equals(ADS_TYPE_URL_CDS_V2)) { @@ -950,7 +943,7 @@ private void handleCdsResponse(DiscoveryResponse cdsResponse) { } catch (InvalidProtocolBufferException e) { logger.log(XdsLogLevel.WARNING, "Failed to unpack Clusters in CDS response {0}", e); adsStream.sendNackRequest( - ADS_TYPE_URL_CDS_V2, clusterWatchers.keySet(), + ResourceType.CDS, clusterWatchers.keySet(), cdsResponse.getVersionInfo(), "Malformed CDS response: " + e); return; } @@ -1025,13 +1018,13 @@ private void handleCdsResponse(DiscoveryResponse cdsResponse) { } if (errorMessage != null) { adsStream.sendNackRequest( - ADS_TYPE_URL_CDS_V2, + ResourceType.CDS, clusterWatchers.keySet(), cdsResponse.getVersionInfo(), errorMessage); return; } - adsStream.sendAckRequest(ADS_TYPE_URL_CDS_V2, clusterWatchers.keySet(), + adsStream.sendAckRequest(ResourceType.CDS, clusterWatchers.keySet(), cdsResponse.getVersionInfo()); // Update local CDS cache with data in this response. @@ -1107,15 +1100,11 @@ private static UpstreamTlsContext getTlsContextFromCluster(Cluster cluster) * cached locally, in case of new endpoint watchers interested in the same clusters * are added later. */ - private void handleEdsResponse(DiscoveryResponse edsResponse) { - if (logger.isLoggable(XdsLogLevel.DEBUG)) { - logger.log(XdsLogLevel.DEBUG, "Received EDS response:\n{0}", respPrinter.print(edsResponse)); - } - + private void handleEdsResponse(DiscoveryResponseData edsResponse) { // Unpack ClusterLoadAssignment messages. List clusterLoadAssignments = - new ArrayList<>(edsResponse.getResourcesCount()); - List claNames = new ArrayList<>(edsResponse.getResourcesCount()); + new ArrayList<>(edsResponse.getResourcesList().size()); + List claNames = new ArrayList<>(edsResponse.getResourcesList().size()); try { for (com.google.protobuf.Any res : edsResponse.getResourcesList()) { if (res.getTypeUrl().equals(ADS_TYPE_URL_EDS_V2)) { @@ -1129,7 +1118,7 @@ private void handleEdsResponse(DiscoveryResponse edsResponse) { logger.log( XdsLogLevel.WARNING, "Failed to unpack ClusterLoadAssignments in EDS response {0}", e); adsStream.sendNackRequest( - ADS_TYPE_URL_EDS_V2, endpointWatchers.keySet(), + ResourceType.EDS, endpointWatchers.keySet(), edsResponse.getVersionInfo(), "Malformed EDS response: " + e); return; } @@ -1202,13 +1191,13 @@ private void handleEdsResponse(DiscoveryResponse edsResponse) { } if (errorMessage != null) { adsStream.sendNackRequest( - ADS_TYPE_URL_EDS_V2, + ResourceType.EDS, endpointWatchers.keySet(), edsResponse.getVersionInfo(), errorMessage); return; } - adsStream.sendAckRequest(ADS_TYPE_URL_EDS_V2, endpointWatchers.keySet(), + adsStream.sendAckRequest(ResourceType.EDS, endpointWatchers.keySet(), edsResponse.getVersionInfo()); // Update local EDS cache by inserting updated endpoint information. @@ -1239,7 +1228,7 @@ final class RpcRetryTask implements Runnable { public void run() { startRpcStream(); if (configWatcher != null) { - adsStream.sendXdsRequest(ADS_TYPE_URL_LDS_V2, ImmutableList.of(ldsResourceName)); + adsStream.sendXdsRequest(ResourceType.LDS, ImmutableList.of(ldsResourceName)); ldsRespTimer = syncContext .schedule( @@ -1247,7 +1236,7 @@ public void run() { INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); } if (listenerWatcher != null) { - adsStream.sendXdsRequest(ADS_TYPE_URL_LDS_V2, ImmutableList.of()); + adsStream.sendXdsRequest(ResourceType.LDS, ImmutableList.of()); ldsRespTimer = syncContext .schedule( @@ -1255,7 +1244,7 @@ public void run() { INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); } if (!clusterWatchers.isEmpty()) { - adsStream.sendXdsRequest(ADS_TYPE_URL_CDS_V2, clusterWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.CDS, clusterWatchers.keySet()); for (String clusterName : clusterWatchers.keySet()) { ScheduledHandle timeoutHandle = syncContext @@ -1266,7 +1255,7 @@ public void run() { } } if (!endpointWatchers.isEmpty()) { - adsStream.sendXdsRequest(ADS_TYPE_URL_EDS_V2, endpointWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.EDS, endpointWatchers.keySet()); for (String clusterName : endpointWatchers.keySet()) { ScheduledHandle timeoutHandle = syncContext @@ -1279,10 +1268,159 @@ public void run() { } } - private final class AdsStream implements StreamObserver { - private final AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub stub; + private enum ResourceType { + UNKNOWN, LDS, RDS, CDS, EDS; + + String typeUrl() { + switch (this) { + case LDS: + return ADS_TYPE_URL_LDS; + case RDS: + return ADS_TYPE_URL_RDS; + case CDS: + return ADS_TYPE_URL_CDS; + case EDS: + return ADS_TYPE_URL_EDS; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + this); + } + } + + String typeUrlV2() { + switch (this) { + case LDS: + return ADS_TYPE_URL_LDS_V2; + case RDS: + return ADS_TYPE_URL_RDS_V2; + case CDS: + return ADS_TYPE_URL_CDS_V2; + case EDS: + return ADS_TYPE_URL_EDS_V2; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + this); + } + } + + static ResourceType fromTypeUrl(String typeUrl) { + switch (typeUrl) { + case ADS_TYPE_URL_LDS: + // fall trough + case ADS_TYPE_URL_LDS_V2: + return LDS; + case ADS_TYPE_URL_RDS: + // fall through + case ADS_TYPE_URL_RDS_V2: + return RDS; + case ADS_TYPE_URL_CDS: + // fall through + case ADS_TYPE_URL_CDS_V2: + return CDS; + case ADS_TYPE_URL_EDS: + // fall through + case ADS_TYPE_URL_EDS_V2: + return EDS; + default: + return UNKNOWN; + } + } + } - private StreamObserver requestWriter; + private static final class DiscoveryRequestData { + private final ResourceType resourceType; + private final Collection resourceNames; + private final String versionInfo; + private final String responseNonce; + private final Node node; + @Nullable + private final com.google.rpc.Status errorDetail; + + DiscoveryRequestData( + ResourceType resourceType, Collection resourceNames, String versionInfo, + String responseNonce, Node node, @Nullable com.google.rpc.Status errorDetail) { + this.resourceType = resourceType; + this.resourceNames = resourceNames; + this.versionInfo = versionInfo; + this.responseNonce = responseNonce; + this.node = node; + this.errorDetail = errorDetail; + } + + DiscoveryRequest toEnvoyProto() { + DiscoveryRequest.Builder builder = + DiscoveryRequest.newBuilder() + .setVersionInfo(versionInfo) + .setNode(node.toEnvoyProtoNode()) + .addAllResourceNames(resourceNames) + .setTypeUrl(resourceType.typeUrl()) + .setResponseNonce(responseNonce); + if (errorDetail != null) { + builder.setErrorDetail(errorDetail); + } + return builder.build(); + } + + io.envoyproxy.envoy.api.v2.DiscoveryRequest toEnvoyProtoV2() { + io.envoyproxy.envoy.api.v2.DiscoveryRequest.Builder builder = + io.envoyproxy.envoy.api.v2.DiscoveryRequest.newBuilder() + .setVersionInfo(versionInfo) + .setNode(node.toEnvoyProtoNodeV2()) + .addAllResourceNames(resourceNames) + .setTypeUrl(resourceType.typeUrlV2()) + .setResponseNonce(responseNonce); + if (errorDetail != null) { + builder.setErrorDetail(errorDetail); + } + return builder.build(); + } + } + + private static final class DiscoveryResponseData { + private final ResourceType resourceType; + private final List resources; + private final String versionInfo; + private final String nonce; + + DiscoveryResponseData( + ResourceType resourceType, List resources, String versionInfo, String nonce) { + this.resourceType = resourceType; + this.resources = resources; + this.versionInfo = versionInfo; + this.nonce = nonce; + } + + ResourceType getResourceType() { + return resourceType; + } + + List getResourcesList() { + return resources; + } + + String getVersionInfo() { + return versionInfo; + } + + String getNonce() { + return nonce; + } + + static DiscoveryResponseData fromEnvoyProto(DiscoveryResponse proto) { + return new DiscoveryResponseData( + ResourceType.fromTypeUrl(proto.getTypeUrl()), proto.getResourcesList(), + proto.getVersionInfo(), proto.getNonce()); + } + + static DiscoveryResponseData fromEnvoyProtoV2( + io.envoyproxy.envoy.api.v2.DiscoveryResponse proto) { + return new DiscoveryResponseData( + ResourceType.fromTypeUrl(proto.getTypeUrl()), proto.getResourcesList(), + proto.getVersionInfo(), proto.getNonce()); + } + } + + private abstract class AbstractAdsStream { private boolean responseReceived; private boolean closed; @@ -1313,51 +1451,58 @@ private final class AdsStream implements StreamObserver { @Nullable private String rdsResourceName; - private AdsStream(AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub stub) { - this.stub = checkNotNull(stub, "stub"); - } - - private void start() { - requestWriter = stub.withWaitForReady().streamAggregatedResources(this); - } - - @Override - public void onNext(final DiscoveryResponse response) { - syncContext.execute(new Runnable() { - @Override - public void run() { - if (closed) { - return; - } - responseReceived = true; - String typeUrl = response.getTypeUrl(); - // Nonce in each response is echoed back in the following ACK/NACK request. It is - // used for management server to identify which response the client is ACKing/NACking. - // To avoid confusion, client-initiated requests will always use the nonce in - // most recently received responses of each resource type. - if (typeUrl.equals(ADS_TYPE_URL_LDS_V2) || typeUrl.equals(ADS_TYPE_URL_LDS)) { - ldsRespNonce = response.getNonce(); - handleLdsResponse(response); - } else if (typeUrl.equals(ADS_TYPE_URL_RDS_V2) || typeUrl.equals(ADS_TYPE_URL_LDS)) { - rdsRespNonce = response.getNonce(); - handleRdsResponse(response); - } else if (typeUrl.equals(ADS_TYPE_URL_CDS_V2) || typeUrl.equals(ADS_TYPE_URL_CDS)) { - cdsRespNonce = response.getNonce(); - handleCdsResponse(response); - } else if (typeUrl.equals(ADS_TYPE_URL_EDS_V2) || typeUrl.equals(ADS_TYPE_URL_EDS)) { - edsRespNonce = response.getNonce(); - handleEdsResponse(response); - } else { - logger.log( - XdsLogLevel.WARNING, - "Received an unknown type of DiscoveryResponse\n{0}", response); - } - } - }); - } - - @Override - public void onError(final Throwable t) { + abstract void start(); + + abstract void sendDiscoveryRequest(DiscoveryRequestData request); + + abstract void sendError(Exception error); + + final void onDiscoveryResponse(final DiscoveryResponseData response) { + syncContext.execute( + new Runnable() { + @Override + public void run() { + if (closed) { + return; + } + responseReceived = true; + String respNonce = response.getNonce(); + // Nonce in each response is echoed back in the following ACK/NACK request. It is + // used for management server to identify which response the client is ACKing/NACking. + // To avoid confusion, client-initiated requests will always use the nonce in + // most recently received responses of each resource type. + ResourceType resourceType = response.getResourceType(); + switch (resourceType) { + case LDS: + ldsRespNonce = respNonce; + handleLdsResponse(response); + break; + case RDS: + rdsRespNonce = respNonce; + handleRdsResponse(response); + break; + case CDS: + cdsRespNonce = respNonce; + handleCdsResponse(response); + break; + case EDS: + edsRespNonce = respNonce; + handleEdsResponse(response); + break; + case UNKNOWN: + logger.log( + XdsLogLevel.WARNING, + "Received an unknown type of DiscoveryResponse\n{0}", + respNonce); + break; + default: + throw new AssertionError("Missing case in enum switch: " + resourceType); + } + } + }); + } + + final void onError(final Throwable t) { syncContext.execute(new Runnable() { @Override public void run() { @@ -1366,8 +1511,7 @@ public void run() { }); } - @Override - public void onCompleted() { + final void onCompleted() { syncContext.execute(new Runnable() { @Override public void run() { @@ -1430,7 +1574,7 @@ private void close(Exception error) { } closed = true; cleanUp(); - requestWriter.onError(error); + sendError(error); } private void cleanUp() { @@ -1444,126 +1588,245 @@ private void cleanUp() { * requested resource name (except for LDS as we always request for the singleton Listener) * as we need it to find resources in responses. */ - private void sendXdsRequest(String typeUrl, Collection resourceNames) { - checkState(requestWriter != null, "ADS stream has not been started"); - String version = ""; - String nonce = ""; - if (typeUrl.equals(ADS_TYPE_URL_LDS_V2)) { - version = ldsVersion; - nonce = ldsRespNonce; - logger.log(XdsLogLevel.INFO, "Sending LDS request for resources: {0}", resourceNames); - } else if (typeUrl.equals(ADS_TYPE_URL_RDS_V2)) { - checkArgument(resourceNames.size() == 1, - "RDS request requesting for more than one resource"); - version = rdsVersion; - nonce = rdsRespNonce; - rdsResourceName = resourceNames.iterator().next(); - logger.log(XdsLogLevel.INFO, "Sending RDS request for resources: {0}", resourceNames); - } else if (typeUrl.equals(ADS_TYPE_URL_CDS_V2)) { - version = cdsVersion; - nonce = cdsRespNonce; - logger.log(XdsLogLevel.INFO, "Sending CDS request for resources: {0}", resourceNames); - } else if (typeUrl.equals(ADS_TYPE_URL_EDS_V2)) { - version = edsVersion; - nonce = edsRespNonce; - logger.log(XdsLogLevel.INFO, "Sending EDS request for resources: {0}", resourceNames); - } - DiscoveryRequest request = - DiscoveryRequest - .newBuilder() - .setVersionInfo(version) - .setNode(node.toEnvoyProtoNodeV2()) - .addAllResourceNames(resourceNames) - .setTypeUrl(typeUrl) - .setResponseNonce(nonce) - .build(); - requestWriter.onNext(request); - logger.log(XdsLogLevel.DEBUG, "Sent DiscoveryRequest\n{0}", request); + private void sendXdsRequest(ResourceType resourceType, Collection resourceNames) { + String version; + String nonce; + switch (resourceType) { + case LDS: + version = ldsVersion; + nonce = ldsRespNonce; + logger.log(XdsLogLevel.INFO, "Sending LDS request for resources: {0}", resourceNames); + break; + case RDS: + checkArgument( + resourceNames.size() == 1, "RDS request requesting for more than one resource"); + version = rdsVersion; + nonce = rdsRespNonce; + rdsResourceName = resourceNames.iterator().next(); + logger.log(XdsLogLevel.INFO, "Sending RDS request for resources: {0}", resourceNames); + break; + case CDS: + version = cdsVersion; + nonce = cdsRespNonce; + logger.log(XdsLogLevel.INFO, "Sending CDS request for resources: {0}", resourceNames); + break; + case EDS: + version = edsVersion; + nonce = edsRespNonce; + logger.log(XdsLogLevel.INFO, "Sending EDS request for resources: {0}", resourceNames); + break; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + resourceType); + } + DiscoveryRequestData request = + new DiscoveryRequestData(resourceType, resourceNames, version, nonce, node, null); + sendDiscoveryRequest(request); } /** * Sends a DiscoveryRequest with the given information as an ACK. Updates the latest accepted * version for the corresponding resource type. */ - private void sendAckRequest(String typeUrl, Collection resourceNames, + private void sendAckRequest(ResourceType resourceType, Collection resourceNames, String versionInfo) { - checkState(requestWriter != null, "ADS stream has not been started"); - String nonce = ""; - if (typeUrl.equals(ADS_TYPE_URL_LDS_V2)) { - ldsVersion = versionInfo; - nonce = ldsRespNonce; - } else if (typeUrl.equals(ADS_TYPE_URL_RDS_V2)) { - rdsVersion = versionInfo; - nonce = rdsRespNonce; - } else if (typeUrl.equals(ADS_TYPE_URL_CDS_V2)) { - cdsVersion = versionInfo; - nonce = cdsRespNonce; - } else if (typeUrl.equals(ADS_TYPE_URL_EDS_V2)) { - edsVersion = versionInfo; - nonce = edsRespNonce; - } - DiscoveryRequest request = - DiscoveryRequest - .newBuilder() - .setVersionInfo(versionInfo) - .setNode(node.toEnvoyProtoNodeV2()) - .addAllResourceNames(resourceNames) - .setTypeUrl(typeUrl) - .setResponseNonce(nonce) - .build(); - requestWriter.onNext(request); - logger.log(XdsLogLevel.DEBUG, "Sent ACK request\n{0}", request); + String nonce; + switch (resourceType) { + case LDS: + ldsVersion = versionInfo; + nonce = ldsRespNonce; + logger.log(XdsLogLevel.WARNING, "Sending ACK for LDS update, version: {0}", versionInfo); + break; + case RDS: + rdsVersion = versionInfo; + nonce = rdsRespNonce; + logger.log(XdsLogLevel.WARNING, "Sending ACK for RDS update, version: {0}", versionInfo); + break; + case CDS: + cdsVersion = versionInfo; + nonce = cdsRespNonce; + logger.log(XdsLogLevel.WARNING, "Sending ACK for CDS update, version: {0}", versionInfo); + break; + case EDS: + edsVersion = versionInfo; + nonce = edsRespNonce; + logger.log(XdsLogLevel.WARNING, "Sending ACK for EDS update, version: {0}", versionInfo); + break; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + resourceType); + } + DiscoveryRequestData request = + new DiscoveryRequestData(resourceType, resourceNames, versionInfo, nonce, node, null); + sendDiscoveryRequest(request); } /** * Sends a DiscoveryRequest with the given information as an NACK. NACK takes the previous * accepted version. */ - private void sendNackRequest(String typeUrl, Collection resourceNames, + private void sendNackRequest(ResourceType resourceType, Collection resourceNames, String rejectVersion, String message) { + String versionInfo; + String nonce; + switch (resourceType) { + case LDS: + versionInfo = ldsVersion; + nonce = ldsRespNonce; + logger.log( + XdsLogLevel.WARNING, + "Sending NACK for LDS update, version: {0}, reason: {1}", + rejectVersion, + message); + break; + case RDS: + versionInfo = rdsVersion; + nonce = rdsRespNonce; + logger.log( + XdsLogLevel.WARNING, + "Sending NACK for RDS update, version: {0}, reason: {1}", + rejectVersion, + message); + break; + case CDS: + versionInfo = cdsVersion; + nonce = cdsRespNonce; + logger.log( + XdsLogLevel.WARNING, + "Sending NACK for CDS update, version: {0}, reason: {1}", + rejectVersion, + message); + break; + case EDS: + versionInfo = edsVersion; + nonce = edsRespNonce; + logger.log( + XdsLogLevel.WARNING, + "Sending NACK for EDS update, version: {0}, reason: {1}", + rejectVersion, + message); + break; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + resourceType); + } + com.google.rpc.Status error = com.google.rpc.Status.newBuilder() + .setCode(Code.INVALID_ARGUMENT_VALUE) + .setMessage(message) + .build(); + DiscoveryRequestData request = + new DiscoveryRequestData(resourceType, resourceNames, versionInfo, nonce, node, error); + sendDiscoveryRequest(request); + } + } + + private final class AdsStreamV2 extends AbstractAdsStream { + private final io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc + .AggregatedDiscoveryServiceStub stubV2; + private StreamObserver requestWriterV2; + + AdsStreamV2(io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc + .AggregatedDiscoveryServiceStub stubV2) { + this.stubV2 = checkNotNull(stubV2, "stubV2"); + } + + @Override + void start() { + StreamObserver responseReaderV2 = + new StreamObserver() { + @Override + public void onNext(io.envoyproxy.envoy.api.v2.DiscoveryResponse response) { + DiscoveryResponseData responseData = + DiscoveryResponseData.fromEnvoyProtoV2(response); + if (logger.isLoggable(XdsLogLevel.DEBUG)) { + logger.log( + XdsLogLevel.DEBUG, + "Received {0} response:\n{1}", + responseData.getResourceType(), + respPrinter.print(response)); + } + onDiscoveryResponse(responseData); + } + + @Override + public void onError(Throwable t) { + AdsStreamV2.this.onError(t); + } + + @Override + public void onCompleted() { + AdsStreamV2.this.onCompleted(); + } + }; + requestWriterV2 = stubV2.withWaitForReady().streamAggregatedResources(responseReaderV2); + } + + @Override + void sendDiscoveryRequest(DiscoveryRequestData request) { + checkState(requestWriterV2 != null, "ADS stream has not been started"); + io.envoyproxy.envoy.api.v2.DiscoveryRequest requestProto = + request.toEnvoyProtoV2(); + requestWriterV2.onNext(requestProto); + logger.log(XdsLogLevel.DEBUG, "Sent DiscoveryRequest\n{0}", requestProto); + } + + @Override + void sendError(Exception error) { + requestWriterV2.onError(error); + } + } + + // AdsStream V3 + @SuppressWarnings("UnusedNestedClass") // Will be used once xds-v3 support is implemented. + private final class AdsStream extends AbstractAdsStream { + private final AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub stub; + private StreamObserver requestWriter; + + AdsStream(AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub stub) { + this.stub = checkNotNull(stub, "stub"); + } + + @Override + void start() { + StreamObserver responseReader = new StreamObserver() { + @Override + public void onNext(DiscoveryResponse response) { + DiscoveryResponseData responseData = + DiscoveryResponseData.fromEnvoyProto(response); + if (logger.isLoggable(XdsLogLevel.DEBUG)) { + logger.log( + XdsLogLevel.DEBUG, + "Received {0} response:\n{1}", + responseData.getResourceType(), + respPrinter.print(response)); + } + onDiscoveryResponse(responseData); + } + + @Override + public void onError(Throwable t) { + AdsStream.this.onError(t); + } + + @Override + public void onCompleted() { + AdsStream.this.onCompleted(); + } + }; + requestWriter = stub.withWaitForReady().streamAggregatedResources(responseReader); + } + + @Override + void sendDiscoveryRequest(DiscoveryRequestData request) { checkState(requestWriter != null, "ADS stream has not been started"); - String versionInfo = ""; - String nonce = ""; - if (typeUrl.equals(ADS_TYPE_URL_LDS_V2)) { - versionInfo = ldsVersion; - nonce = ldsRespNonce; - logger.log( - XdsLogLevel.WARNING, - "Rejecting LDS update, version: {0}, reason: {1}", rejectVersion, message); - } else if (typeUrl.equals(ADS_TYPE_URL_RDS_V2)) { - versionInfo = rdsVersion; - nonce = rdsRespNonce; - logger.log( - XdsLogLevel.WARNING, - "Rejecting RDS update, version: {0}, reason: {1}", rejectVersion, message); - } else if (typeUrl.equals(ADS_TYPE_URL_CDS_V2)) { - versionInfo = cdsVersion; - nonce = cdsRespNonce; - logger.log( - XdsLogLevel.WARNING, - "Rejecting CDS update, version: {0}, reason: {1}", rejectVersion, message); - } else if (typeUrl.equals(ADS_TYPE_URL_EDS_V2)) { - versionInfo = edsVersion; - nonce = edsRespNonce; - logger.log( - XdsLogLevel.WARNING, - "Rejecting EDS update, version: {0}, reason: {1}", rejectVersion, message); - } - DiscoveryRequest request = - DiscoveryRequest - .newBuilder() - .setVersionInfo(versionInfo) - .setNode(node.toEnvoyProtoNodeV2()) - .addAllResourceNames(resourceNames) - .setTypeUrl(typeUrl) - .setResponseNonce(nonce) - .setErrorDetail( - com.google.rpc.Status.newBuilder() - .setCode(Code.INVALID_ARGUMENT_VALUE) - .setMessage(message)) - .build(); - requestWriter.onNext(request); - logger.log(XdsLogLevel.DEBUG, "Sent NACK request\n{0}", request); + DiscoveryRequest requestProto = request.toEnvoyProto(); + requestWriter.onNext(requestProto); + logger.log(XdsLogLevel.DEBUG, "Sent DiscoveryRequest\n{0}", requestProto); + } + + @Override + void sendError(Exception error) { + requestWriter.onError(error); } } From 588d6b66b2fca4aa79a211ffc048dfbeac028cda Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Thu, 30 Jul 2020 18:00:37 -0700 Subject: [PATCH 5/9] xds: Add server features support to Bootstrapper In preparation for xds-v3 support. --- .../main/java/io/grpc/xds/Bootstrapper.java | 18 +++++- xds/src/main/java/io/grpc/xds/XdsClient.java | 35 +++++++++-- .../main/java/io/grpc/xds/XdsClientImpl.java | 3 +- .../java/io/grpc/xds/BootstrapperTest.java | 4 ++ .../java/io/grpc/xds/EdsLoadBalancerTest.java | 10 ++-- .../java/io/grpc/xds/XdsClientImplTest.java | 12 ++-- .../xds/XdsClientImplTestForListener.java | 12 ++-- .../test/java/io/grpc/xds/XdsClientTest.java | 59 +++++++++++++++++++ .../xds/XdsNameResolverIntegrationTest.java | 9 ++- 9 files changed, 134 insertions(+), 28 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/Bootstrapper.java b/xds/src/main/java/io/grpc/xds/Bootstrapper.java index f967d43d8fd..6fa3c4e5710 100644 --- a/xds/src/main/java/io/grpc/xds/Bootstrapper.java +++ b/xds/src/main/java/io/grpc/xds/Bootstrapper.java @@ -111,7 +111,11 @@ static BootstrapInfo parseConfig(String rawData) throws IOException { channelCredsOptions.add(creds); } } - servers.add(new ServerInfo(serverUri, channelCredsOptions)); + List serverFeatures = JsonUtil.getListOfStrings(serverConfig, "server_features"); + if (serverFeatures != null) { + logger.log(XdsLogLevel.INFO, "Server features: {0}", serverFeatures); + } + servers.add(new ServerInfo(serverUri, channelCredsOptions, serverFeatures)); } Node.Builder nodeBuilder = Node.newBuilder(); @@ -196,11 +200,14 @@ String getType() { static class ServerInfo { private final String serverUri; private final List channelCredsList; + @Nullable + private final List serverFeatures; @VisibleForTesting - ServerInfo(String serverUri, List channelCredsList) { + ServerInfo(String serverUri, List channelCredsList, List serverFeatures) { this.serverUri = serverUri; this.channelCredsList = channelCredsList; + this.serverFeatures = serverFeatures; } String getServerUri() { @@ -210,6 +217,12 @@ String getServerUri() { List getChannelCredentials() { return Collections.unmodifiableList(channelCredsList); } + + List getServerFeatures() { + return serverFeatures == null + ? Collections.emptyList() + : Collections.unmodifiableList(serverFeatures); + } } /** @@ -240,6 +253,5 @@ List getServers() { public Node getNode() { return node; } - } } diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index b74ec3360b3..1c895c878ab 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -601,13 +601,17 @@ public synchronized XdsClient returnObject(Object object) { * Factory for creating channels to xDS severs. */ abstract static class XdsChannelFactory { - private static final XdsChannelFactory DEFAULT_INSTANCE = new XdsChannelFactory() { + @VisibleForTesting + static boolean experimentalV3SupportEnvVar = Boolean.parseBoolean( + System.getenv("GRPC_XDS_EXPERIMENTAL_V3_SUPPORT")); + private static final String XDS_V3_SERVER_FEATURE = "xds_v3"; + private static final XdsChannelFactory DEFAULT_INSTANCE = new XdsChannelFactory() { /** * Creates a channel to the first server in the given list. */ @Override - ManagedChannel createChannel(List servers) { + XdsChannel createChannel(List servers) { checkArgument(!servers.isEmpty(), "No management server provided."); XdsLogger logger = XdsLogger.withPrefix("xds-client-channel-factory"); ServerInfo serverInfo = servers.get(0); @@ -629,9 +633,13 @@ ManagedChannel createChannel(List servers) { channelBuilder = ManagedChannelBuilder.forTarget(serverUri); } - return channelBuilder + ManagedChannel channel = channelBuilder .keepAliveTime(5, TimeUnit.MINUTES) .build(); + boolean useProtocolV3 = experimentalV3SupportEnvVar + && serverInfo.getServerFeatures().contains(XDS_V3_SERVER_FEATURE); + + return new XdsChannel(channel, useProtocolV3); } }; @@ -642,6 +650,25 @@ static XdsChannelFactory getInstance() { /** * Creates a channel to one of the provided management servers. */ - abstract ManagedChannel createChannel(List servers); + abstract XdsChannel createChannel(List servers); + } + + static final class XdsChannel { + private final ManagedChannel managedChannel; + private final boolean useProtocolV3; + + @VisibleForTesting + XdsChannel(ManagedChannel managedChannel, boolean useProtocolV3) { + this.managedChannel = managedChannel; + this.useProtocolV3 = useProtocolV3; + } + + ManagedChannel getManagedChannel() { + return managedChannel; + } + + boolean isUseProtocolV3() { + return useProtocolV3; + } } } diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index 4991cdf1d83..3b972c2b30d 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -200,7 +200,8 @@ final class XdsClientImpl extends XdsClient { this.targetName = checkNotNull(targetName, "targetName"); this.channel = checkNotNull(channelFactory, "channelFactory") - .createChannel(checkNotNull(servers, "servers")); + .createChannel(checkNotNull(servers, "servers")) + .getManagedChannel(); this.node = checkNotNull(node, "node"); this.syncContext = checkNotNull(syncContext, "syncContext"); this.timeService = checkNotNull(timeService, "timeService"); diff --git a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java index 10d2c109167..6c1fd2111ba 100644 --- a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java @@ -112,6 +112,9 @@ public void parseBootstrap_validData_multipleXdsServers() throws IOException { + " \"server_uri\": \"trafficdirector-foo.googleapis.com:443\",\n" + " \"channel_creds\": [\n" + " {\"type\": \"tls\"}, {\"type\": \"loas\"}, {\"type\": \"google_default\"}\n" + + " ],\n" + + " \"server_features\": [\n" + + " \"xds_v3\", \"foo\", \"bar\"\n" + " ]\n" + " },\n" + " {\n" @@ -134,6 +137,7 @@ public void parseBootstrap_validData_multipleXdsServers() throws IOException { assertThat(serverInfoList.get(0).getChannelCredentials().get(2).getType()) .isEqualTo("google_default"); assertThat(serverInfoList.get(0).getChannelCredentials().get(2).getConfig()).isNull(); + assertThat(serverInfoList.get(0).getServerFeatures()).contains("xds_v3"); assertThat(serverInfoList.get(1).getServerUri()) .isEqualTo("trafficdirector-bar.googleapis.com:443"); assertThat(serverInfoList.get(1).getChannelCredentials()).isEmpty(); diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java index 66ce45c3ff4..a321933f867 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java @@ -78,6 +78,7 @@ import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.LocalityStore.LocalityStoreFactory; import io.grpc.xds.XdsClient.EndpointUpdate; +import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClient.XdsChannelFactory; import java.net.InetSocketAddress; import java.util.ArrayDeque; @@ -135,10 +136,10 @@ public void uncaughtException(Thread t, Throwable e) { private final Map childBalancers = new HashMap<>(); private final XdsChannelFactory channelFactory = new XdsChannelFactory() { @Override - ManagedChannel createChannel(List servers) { + XdsChannel createChannel(List servers) { assertThat(Iterables.getOnlyElement(servers).getServerUri()) .isEqualTo("trafficdirector.googleapis.com"); - return channel; + return new XdsChannel(channel, false); } }; @@ -228,9 +229,8 @@ public StreamObserver streamAggregatedResources( .forName(serverName) .directExecutor() .build()); - final List serverList = - ImmutableList.of( - new ServerInfo("trafficdirector.googleapis.com", ImmutableList.of())); + final List serverList = ImmutableList.of( + new ServerInfo("trafficdirector.googleapis.com", ImmutableList.of(), null)); Node node = Node.newBuilder().build(); BootstrapInfo bootstrapInfo = new BootstrapInfo(serverList, node); doReturn(bootstrapInfo).when(bootstrapper).readBootstrap(); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index a63b5401c03..19c3ec30bee 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -100,6 +100,7 @@ import io.grpc.xds.XdsClient.ConfigWatcher; import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.EndpointWatcher; +import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClient.XdsChannelFactory; import io.grpc.xds.XdsClientImpl.MessagePrinter; import java.io.IOException; @@ -279,13 +280,14 @@ public void cancelled(Context context) { cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); List servers = - ImmutableList.of(new ServerInfo(serverName, ImmutableList.of())); + ImmutableList.of(new ServerInfo(serverName, ImmutableList.of(), null)); XdsChannelFactory channelFactory = new XdsChannelFactory() { @Override - ManagedChannel createChannel(List servers) { - assertThat(Iterables.getOnlyElement(servers).getServerUri()).isEqualTo(serverName); - assertThat(Iterables.getOnlyElement(servers).getChannelCredentials()).isEmpty(); - return channel; + XdsChannel createChannel(List servers) { + ServerInfo serverInfo = Iterables.getOnlyElement(servers); + assertThat(serverInfo.getServerUri()).isEqualTo(serverName); + assertThat(serverInfo.getChannelCredentials()).isEmpty(); + return new XdsChannel(channel, false); } }; diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java index 91d22c69ee0..d2452c86239 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java @@ -71,6 +71,7 @@ import io.grpc.xds.XdsClient.ConfigWatcher; import io.grpc.xds.XdsClient.ListenerUpdate; import io.grpc.xds.XdsClient.ListenerWatcher; +import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClient.XdsChannelFactory; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import java.io.IOException; @@ -201,13 +202,14 @@ public void cancelled(Context context) { cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); List servers = - ImmutableList.of(new ServerInfo(serverName, ImmutableList.of())); + ImmutableList.of(new ServerInfo(serverName, ImmutableList.of(), null)); XdsChannelFactory channelFactory = new XdsChannelFactory() { @Override - ManagedChannel createChannel(List servers) { - assertThat(Iterables.getOnlyElement(servers).getServerUri()).isEqualTo(serverName); - assertThat(Iterables.getOnlyElement(servers).getChannelCredentials()).isEmpty(); - return channel; + XdsChannel createChannel(List servers) { + ServerInfo serverInfo = Iterables.getOnlyElement(servers); + assertThat(serverInfo.getServerUri()).isEqualTo(serverName); + assertThat(serverInfo.getChannelCredentials()).isEmpty(); + return new XdsChannel(channel, false); } }; diff --git a/xds/src/test/java/io/grpc/xds/XdsClientTest.java b/xds/src/test/java/io/grpc/xds/XdsClientTest.java index 56fca6dc09b..9278a80dbb5 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientTest.java @@ -21,7 +21,12 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import com.google.common.collect.ImmutableList; +import io.grpc.xds.Bootstrapper.ChannelCreds; +import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; +import io.grpc.xds.XdsClient.XdsChannel; +import io.grpc.xds.XdsClient.XdsChannelFactory; import io.grpc.xds.XdsClient.XdsClientFactory; import org.junit.Rule; import org.junit.Test; @@ -102,4 +107,58 @@ XdsClient createXdsClient() { XdsClient xdsClient2 = xdsClientPool.getObject(); assertThat(xdsClient2).isNotSameInstanceAs(xdsClient1); } + + @Test + public void channelFactorySupportsV3() { + boolean originalV3SupportEnvVar = XdsChannelFactory.experimentalV3SupportEnvVar; + try { + XdsChannelFactory xdsChannelFactory = XdsChannelFactory.getInstance(); + XdsChannelFactory.experimentalV3SupportEnvVar = true; + XdsChannel xdsChannel = + xdsChannelFactory.createChannel( + ImmutableList.of( + new ServerInfo( + "xdsserver.com", + ImmutableList.of(), + ImmutableList.of()), + new ServerInfo( + "xdsserver2.com", + ImmutableList.of(), + ImmutableList.of("xds_v3")))); + xdsChannel.getManagedChannel().shutdown(); + assertThat(xdsChannel.isUseProtocolV3()).isFalse(); + + XdsChannelFactory.experimentalV3SupportEnvVar = false; + xdsChannel = + xdsChannelFactory.createChannel( + ImmutableList.of( + new ServerInfo( + "xdsserver.com", + ImmutableList.of(), + ImmutableList.of("xds_v3")), + new ServerInfo( + "xdsserver2.com", + ImmutableList.of(), + ImmutableList.of("baz")))); + xdsChannel.getManagedChannel().shutdown(); + assertThat(xdsChannel.isUseProtocolV3()).isFalse(); + + XdsChannelFactory.experimentalV3SupportEnvVar = true; + xdsChannel = + xdsChannelFactory.createChannel( + ImmutableList.of( + new ServerInfo( + "xdsserver.com", + ImmutableList.of(), + ImmutableList.of("xds_v3")), + new ServerInfo( + "xdsserver2.com", + ImmutableList.of(), + ImmutableList.of("baz")))); + xdsChannel.getManagedChannel().shutdown(); + assertThat(xdsChannel.isUseProtocolV3()).isTrue(); + } finally { + XdsChannelFactory.experimentalV3SupportEnvVar = originalV3SupportEnvVar; + } + } } diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java index 4584bcc6ea7..804c50f5106 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java @@ -65,6 +65,7 @@ import io.grpc.testing.GrpcCleanupRule; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.Node; +import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClient.XdsChannelFactory; import java.io.IOException; import java.util.ArrayDeque; @@ -159,18 +160,16 @@ public StreamObserver streamAggregatedResources( channelFactory = new XdsChannelFactory() { @Override - ManagedChannel createChannel(List servers) { + XdsChannel createChannel(List servers) { assertThat(Iterables.getOnlyElement(servers).getServerUri()).isEqualTo(serverName); - return channel; + return new XdsChannel(channel, false); } }; Bootstrapper bootstrapper = new Bootstrapper() { @Override public BootstrapInfo readBootstrap() { List serverList = - ImmutableList.of( - new ServerInfo(serverName, - ImmutableList.of())); + ImmutableList.of(new ServerInfo(serverName, ImmutableList.of(), null)); return new BootstrapInfo(serverList, FAKE_BOOTSTRAP_NODE); } }; From c1f6335d010d75622d60903a7a96f8302b5d5631 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 31 Jul 2020 19:12:00 +0000 Subject: [PATCH 6/9] xds: parse timeout from RDS responses (#7257) --- .../main/java/io/grpc/xds/EnvoyProtoData.java | 31 ++++++++++-- .../java/io/grpc/xds/EnvoyProtoDataTest.java | 48 +++++++++++++++---- .../java/io/grpc/xds/XdsClientImplTest.java | 9 ++-- .../java/io/grpc/xds/XdsNameResolverTest.java | 5 +- 4 files changed, 74 insertions(+), 19 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java index eeae2f96174..14a62f29f5a 100644 --- a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java +++ b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java @@ -26,6 +26,7 @@ import com.google.protobuf.NullValue; import com.google.protobuf.Struct; import com.google.protobuf.Value; +import com.google.protobuf.util.Durations; import com.google.re2j.Pattern; import com.google.re2j.PatternSyntaxException; import io.envoyproxy.envoy.type.v3.FractionalPercent; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; /** @@ -1067,6 +1069,7 @@ static StructOrError convertEnvoyProtoHeaderMatcher( /** See corresponding Envoy proto message {@link io.envoyproxy.envoy.api.v2.route.RouteAction}. */ static final class RouteAction { + private final long timeoutNano; // Exactly one of the following fields is non-null. @Nullable private final String cluster; @@ -1074,11 +1077,20 @@ static final class RouteAction { private final List weightedClusters; @VisibleForTesting - RouteAction(@Nullable String cluster, @Nullable List weightedClusters) { + RouteAction( + long timeoutNano, + @Nullable String cluster, + @Nullable List weightedClusters) { + this.timeoutNano = timeoutNano; this.cluster = cluster; this.weightedClusters = weightedClusters; } + + Long getTimeoutNano() { + return timeoutNano; + } + @Nullable String getCluster() { return cluster; @@ -1098,18 +1110,20 @@ public boolean equals(Object o) { return false; } RouteAction that = (RouteAction) o; - return Objects.equals(cluster, that.cluster) + return Objects.equals(timeoutNano, that.timeoutNano) + && Objects.equals(cluster, that.cluster) && Objects.equals(weightedClusters, that.weightedClusters); } @Override public int hashCode() { - return Objects.hash(cluster, weightedClusters); + return Objects.hash(timeoutNano, cluster, weightedClusters); } @Override public String toString() { ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); + toStringHelper.add("timeout", timeoutNano + "ns"); if (cluster != null) { toStringHelper.add("cluster", cluster); } @@ -1146,7 +1160,16 @@ static StructOrError fromEnvoyProtoRouteAction( return StructOrError.fromError( "Unknown cluster specifier: " + proto.getClusterSpecifierCase()); } - return StructOrError.fromStruct(new RouteAction(cluster, weightedClusters)); + long timeoutNano = TimeUnit.SECONDS.toNanos(15L); // default 15s + if (proto.hasMaxGrpcTimeout()) { + timeoutNano = Durations.toNanos(proto.getMaxGrpcTimeout()); + } else if (proto.hasTimeout()) { + timeoutNano = Durations.toNanos(proto.getTimeout()); + } + if (timeoutNano == 0) { + timeoutNano = Long.MAX_VALUE; + } + return StructOrError.fromStruct(new RouteAction(timeoutNano, cluster, weightedClusters)); } } diff --git a/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java b/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java index fa278fb8147..24e739bf560 100644 --- a/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java +++ b/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java @@ -24,6 +24,7 @@ import com.google.protobuf.Struct; import com.google.protobuf.UInt32Value; import com.google.protobuf.Value; +import com.google.protobuf.util.Durations; import com.google.re2j.Pattern; import io.envoyproxy.envoy.config.core.v3.RuntimeFractionalPercent; import io.envoyproxy.envoy.config.route.v3.QueryParameterMatcher; @@ -44,6 +45,7 @@ import io.grpc.xds.RouteMatch.PathMatcher; import java.util.Arrays; import java.util.Collections; +import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.junit.Test; import org.junit.runner.RunWith; @@ -206,7 +208,7 @@ public void convertRoute() { new Route( new RouteMatch(new PathMatcher("/service/method", null, null), Collections.emptyList(), null), - new RouteAction("cluster-foo", null))); + new RouteAction(TimeUnit.SECONDS.toNanos(15L), "cluster-foo", null))); io.envoyproxy.envoy.config.route.v3.Route unsupportedProto = io.envoyproxy.envoy.config.route.v3.Route.newBuilder() @@ -393,27 +395,51 @@ public void convertRouteMatch_withRuntimeFraction() { @Test public void convertRouteAction() { - // cluster_specifier = cluster + // cluster_specifier = cluster, default timeout io.envoyproxy.envoy.config.route.v3.RouteAction proto1 = io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() .setCluster("cluster-foo") .build(); StructOrError struct1 = RouteAction.fromEnvoyProtoRouteAction(proto1); assertThat(struct1.getErrorDetail()).isNull(); + assertThat(struct1.getStruct().getTimeoutNano()) + .isEqualTo(TimeUnit.SECONDS.toNanos(15L)); // default value assertThat(struct1.getStruct().getCluster()).isEqualTo("cluster-foo"); assertThat(struct1.getStruct().getWeightedCluster()).isNull(); - // cluster_specifier = cluster_header + // cluster_specifier = cluster, infinity timeout io.envoyproxy.envoy.config.route.v3.RouteAction proto2 = io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() - .setClusterHeader("cluster-bar") + .setMaxGrpcTimeout(Durations.fromNanos(0)) + .setTimeout(Durations.fromMicros(20L)) + .setCluster("cluster-foo") .build(); StructOrError struct2 = RouteAction.fromEnvoyProtoRouteAction(proto2); - assertThat(struct2).isNull(); + assertThat(struct2.getStruct().getTimeoutNano()) + .isEqualTo(Long.MAX_VALUE); // infinite - // cluster_specifier = weighted_cluster + // cluster_specifier = cluster, infinity timeout io.envoyproxy.envoy.config.route.v3.RouteAction proto3 = io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() + .setTimeout(Durations.fromNanos(0)) + .setCluster("cluster-foo") + .build(); + StructOrError struct3 = RouteAction.fromEnvoyProtoRouteAction(proto3); + assertThat(struct3.getStruct().getTimeoutNano()).isEqualTo(Long.MAX_VALUE); // infinite + + // cluster_specifier = cluster_header + io.envoyproxy.envoy.config.route.v3.RouteAction proto4 = + io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() + .setClusterHeader("cluster-bar") + .build(); + StructOrError struct4 = RouteAction.fromEnvoyProtoRouteAction(proto4); + assertThat(struct4).isNull(); + + // cluster_specifier = weighted_cluster + io.envoyproxy.envoy.config.route.v3.RouteAction proto5 = + io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() + .setMaxGrpcTimeout(Durations.fromSeconds(6L)) + .setTimeout(Durations.fromMicros(20L)) .setWeightedClusters( WeightedCluster.newBuilder() .addClusters( @@ -422,10 +448,12 @@ public void convertRouteAction() { .setName("cluster-baz") .setWeight(UInt32Value.newBuilder().setValue(100)))) .build(); - StructOrError struct3 = RouteAction.fromEnvoyProtoRouteAction(proto3); - assertThat(struct3.getErrorDetail()).isNull(); - assertThat(struct3.getStruct().getCluster()).isNull(); - assertThat(struct3.getStruct().getWeightedCluster()) + StructOrError struct5 = RouteAction.fromEnvoyProtoRouteAction(proto5); + assertThat(struct5.getErrorDetail()).isNull(); + assertThat(struct5.getStruct().getTimeoutNano()) + .isEqualTo(TimeUnit.SECONDS.toNanos(6L)); + assertThat(struct5.getStruct().getCluster()).isNull(); + assertThat(struct5.getStruct().getWeightedCluster()) .containsExactly(new ClusterWeight("cluster-baz", 100)); // cluster_specifier unset diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index 19c3ec30bee..a771d9fb7df 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -750,7 +750,8 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new io.grpc.xds.RouteMatch( /* prefix= */ null, /* path= */ "/service1/method1"), - new EnvoyProtoData.RouteAction("cl1.googleapis.com", null))); + new EnvoyProtoData.RouteAction( + TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); assertThat(routes.get(1)).isEqualTo( new EnvoyProtoData.Route( // path match with weighted cluster route @@ -758,6 +759,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { /* prefix= */ null, /* path= */ "/service2/method2"), new EnvoyProtoData.RouteAction( + TimeUnit.SECONDS.toNanos(15L), null, ImmutableList.of( new EnvoyProtoData.ClusterWeight("cl21.googleapis.com", 30), @@ -769,7 +771,8 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new io.grpc.xds.RouteMatch( /* prefix= */ "/service1/", /* path= */ null), - new EnvoyProtoData.RouteAction("cl1.googleapis.com", null))); + new EnvoyProtoData.RouteAction( + TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); assertThat(routes.get(3)).isEqualTo( new EnvoyProtoData.Route( // default match with cluster route @@ -777,7 +780,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { /* prefix= */ "", /* path= */ null), new EnvoyProtoData.RouteAction( - "cluster.googleapis.com", null))); + TimeUnit.SECONDS.toNanos(15L), "cluster.googleapis.com", null))); } /** diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 584a53a8f6f..16ce37a6e51 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -83,7 +83,7 @@ public void generateXdsRoutingRawConfig() { new RouteMatch( new PathMatcher(null, "", null), Collections.emptyList(), new FractionMatcher(10, 20)), - new RouteAction("cluster-foo", null)); + new RouteAction(15L, "cluster-foo", null)); Route r2 = new Route( new RouteMatch( @@ -92,6 +92,7 @@ public void generateXdsRoutingRawConfig() { new HeaderMatcher(":scheme", "https", null, null, null, null, null, false)), null), new RouteAction( + 15L, null, Arrays.asList( new ClusterWeight("cluster-foo", 20), @@ -134,7 +135,7 @@ public void generateXdsRoutingRawConfig_allowDuplicateMatchers() { new RouteMatch( new PathMatcher("/service/method", null, null), Collections.emptyList(), null), - new RouteAction("cluster-foo", null)); + new RouteAction(15L, "cluster-foo", null)); Map config = XdsNameResolver.generateXdsRoutingRawConfig(Arrays.asList(route, route)); From 9c4c696debe56f378ac1ea147c780e230d2ff3b1 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Fri, 31 Jul 2020 14:14:10 -0700 Subject: [PATCH 7/9] xds: support v3 for XdsClient Duplicated `XdsClientImptTest` for V3. `XdsClientImptTestV2` and all other tests are still using V2. Even for `XdsClientImptTest`, although the protocol is V3, the test xds server still sends V2 resources in its V3 response. --- .../main/java/io/grpc/xds/XdsClientImpl.java | 42 +- .../java/io/grpc/xds/EdsLoadBalancerTest.java | 6 +- .../java/io/grpc/xds/XdsClientImplTest.java | 247 +- .../xds/XdsClientImplTestForListener.java | 21 +- .../java/io/grpc/xds/XdsClientImplTestV2.java | 3763 +++++++++++++++++ .../java/io/grpc/xds/XdsClientTestHelper.java | 33 +- .../xds/XdsNameResolverIntegrationTest.java | 14 +- 7 files changed, 3962 insertions(+), 164 deletions(-) create mode 100644 xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index 3b972c2b30d..621eaa0ce47 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -84,12 +84,14 @@ final class XdsClientImpl extends XdsClient { @VisibleForTesting static final String ADS_TYPE_URL_LDS_V2 = "type.googleapis.com/envoy.api.v2.Listener"; - private static final String ADS_TYPE_URL_LDS = + @VisibleForTesting + static final String ADS_TYPE_URL_LDS = "type.googleapis.com/envoy.config.listener.v3.Listener"; @VisibleForTesting static final String ADS_TYPE_URL_RDS_V2 = "type.googleapis.com/envoy.api.v2.RouteConfiguration"; - private static final String ADS_TYPE_URL_RDS = + @VisibleForTesting + static final String ADS_TYPE_URL_RDS = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"; private static final String TYPE_URL_HTTP_CONNECTION_MANAGER_V2 = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2" @@ -99,12 +101,14 @@ final class XdsClientImpl extends XdsClient { + ".HttpConnectionManager"; @VisibleForTesting static final String ADS_TYPE_URL_CDS_V2 = "type.googleapis.com/envoy.api.v2.Cluster"; - private static final String ADS_TYPE_URL_CDS = + @VisibleForTesting + static final String ADS_TYPE_URL_CDS = "type.googleapis.com/envoy.config.cluster.v3.Cluster"; @VisibleForTesting static final String ADS_TYPE_URL_EDS_V2 = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"; - private static final String ADS_TYPE_URL_EDS = + @VisibleForTesting + static final String ADS_TYPE_URL_EDS = "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"; private final MessagePrinter respPrinter = new MessagePrinter(); @@ -114,6 +118,7 @@ final class XdsClientImpl extends XdsClient { // Name of the target server this gRPC client is trying to talk to. private final String targetName; private final ManagedChannel channel; + private final boolean useProtocolV3; private final SynchronizationContext syncContext; private final ScheduledExecutorService timeService; private final BackoffPolicy.Provider backoffPolicyProvider; @@ -198,10 +203,11 @@ final class XdsClientImpl extends XdsClient { BackoffPolicy.Provider backoffPolicyProvider, Supplier stopwatchSupplier) { this.targetName = checkNotNull(targetName, "targetName"); - this.channel = + XdsChannel xdsChannel = checkNotNull(channelFactory, "channelFactory") - .createChannel(checkNotNull(servers, "servers")) - .getManagedChannel(); + .createChannel(checkNotNull(servers, "servers")); + this.channel = xdsChannel.getManagedChannel(); + this.useProtocolV3 = xdsChannel.isUseProtocolV3(); this.node = checkNotNull(node, "node"); this.syncContext = checkNotNull(syncContext, "syncContext"); this.timeService = checkNotNull(timeService, "timeService"); @@ -516,12 +522,11 @@ public String toString() { */ private void startRpcStream() { checkState(adsStream == null, "Previous adsStream has not been cleared yet"); - io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc - .AggregatedDiscoveryServiceStub - stub = - io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.newStub( - channel); - adsStream = new AdsStreamV2(stub); + if (useProtocolV3) { + adsStream = new AdsStream(); + } else { + adsStream = new AdsStreamV2(); + } adsStream.start(); logger.log(XdsLogLevel.INFO, "ADS stream started"); adsStreamRetryStopwatch.reset().start(); @@ -1726,9 +1731,9 @@ private final class AdsStreamV2 extends AbstractAdsStream { .AggregatedDiscoveryServiceStub stubV2; private StreamObserver requestWriterV2; - AdsStreamV2(io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc - .AggregatedDiscoveryServiceStub stubV2) { - this.stubV2 = checkNotNull(stubV2, "stubV2"); + AdsStreamV2() { + stubV2 = + io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.newStub(channel); } @Override @@ -1778,13 +1783,12 @@ void sendError(Exception error) { } // AdsStream V3 - @SuppressWarnings("UnusedNestedClass") // Will be used once xds-v3 support is implemented. private final class AdsStream extends AbstractAdsStream { private final AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub stub; private StreamObserver requestWriter; - AdsStream(AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub stub) { - this.stub = checkNotNull(stub, "stub"); + AdsStream() { + stub = AggregatedDiscoveryServiceGrpc.newStub(channel); } @Override diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java index a321933f867..0b923b3033e 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java @@ -22,7 +22,7 @@ import static io.grpc.ConnectivityState.READY; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; import static io.grpc.xds.XdsClientTestHelper.buildClusterLoadAssignment; -import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponse; +import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponseV2; import static io.grpc.xds.XdsClientTestHelper.buildDropOverload; import static io.grpc.xds.XdsClientTestHelper.buildLbEndpoint; import static io.grpc.xds.XdsClientTestHelper.buildLocalityLbEndpoints; @@ -680,7 +680,7 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { // The whole cluster is no longer accessible. // Note that EDS resource removal is achieved by CDS resource update. responseObserver.onNext( - buildDiscoveryResponse( + buildDiscoveryResponseV2( String.valueOf(versionIno++), Collections.emptyList(), XdsClientImpl.ADS_TYPE_URL_CDS_V2, @@ -768,7 +768,7 @@ private void deliverResolvedAddresses( private void deliverClusterLoadAssignments(ClusterLoadAssignment clusterLoadAssignment) { responseObserver.onNext( - buildDiscoveryResponse( + buildDiscoveryResponseV2( String.valueOf(versionIno++), ImmutableList.of(Any.pack(clusterLoadAssignment)), XdsClientImpl.ADS_TYPE_URL_EDS_V2, diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index a771d9fb7df..2a353c77f58 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -51,8 +51,6 @@ import com.google.protobuf.util.Durations; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment.Policy; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; import io.envoyproxy.envoy.api.v2.auth.UpstreamTlsContext; import io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource; import io.envoyproxy.envoy.api.v2.core.ConfigSource; @@ -69,7 +67,9 @@ import io.envoyproxy.envoy.config.route.v3.RouteMatch; import io.envoyproxy.envoy.config.route.v3.VirtualHost; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.SdsSecretConfig; -import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; +import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; import io.envoyproxy.envoy.service.load_stats.v2.LoadReportingServiceGrpc.LoadReportingServiceImplBase; import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest; import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsResponse; @@ -126,7 +126,8 @@ import org.mockito.MockitoAnnotations; /** - * Tests for {@link XdsClientImpl}. + * Tests for {@link XdsClientImpl} with xDS v3 protocol. However, the test xDS server still sends + * update with v2 resources for testing compatibility. */ @RunWith(JUnit4.class) public class XdsClientImplTest { @@ -287,7 +288,7 @@ XdsChannel createChannel(List servers) { ServerInfo serverInfo = Iterables.getOnlyElement(servers); assertThat(serverInfo.getServerUri()).isEqualTo(serverName); assertThat(serverInfo.getChannelCredentials()).isEmpty(); - return new XdsChannel(channel, false); + return new XdsChannel(channel, /* useProtocolV3= */ true); } }; @@ -337,7 +338,7 @@ public void ldsResponseWithoutMatchingResource() { // Client sends an LDS request for the host name (with port) to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); @@ -367,7 +368,7 @@ public void ldsResponseWithoutMatchingResource() { // Client sends an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0000"))); verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); verify(configWatcher, never()).onResourceDoesNotExist(TARGET_AUTHORITY); @@ -393,7 +394,7 @@ public void failToFindVirtualHostInLdsResponseInLineRouteConfig() { // Client sends an LDS request for the host name (with port) to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); io.envoyproxy.envoy.api.v2.RouteConfiguration routeConfig = @@ -416,7 +417,7 @@ public void failToFindVirtualHostInLdsResponseInLineRouteConfig() { verify(requestObserver) .onNext( argThat(new DiscoveryRequestMatcher("", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0000"))); verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); verify(configWatcher, never()).onResourceDoesNotExist(TARGET_AUTHORITY); @@ -442,7 +443,7 @@ public void resolveVirtualHostInLdsResponse() { // Client sends an LDS request for the host name (with port) to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); ScheduledTask ldsRespTimer = Iterables.getOnlyElement( fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); @@ -489,7 +490,7 @@ public void resolveVirtualHostInLdsResponse() { // Client sends an ACK request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0000"))); ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); @@ -515,7 +516,7 @@ public void rdsResponseWithoutMatchingResource() { // Client sends an LDS request for the host name (with port) to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); Rds rdsConfig = Rds.newBuilder() @@ -535,12 +536,12 @@ public void rdsResponseWithoutMatchingResource() { // Client sends an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0000"))); // Client sends an (first) RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_RDS, ""))); assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); @@ -567,7 +568,7 @@ public void rdsResponseWithoutMatchingResource() { // Client sends an ACK RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0000"))); verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); verify(configWatcher, never()).onResourceDoesNotExist(anyString()); @@ -633,7 +634,7 @@ public void resolveVirtualHostInRdsResponse() { // Client sent an ACK RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0000"))); ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); @@ -738,7 +739,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { // Client sent an ACK RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0000"))); ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); @@ -839,7 +840,7 @@ public void failToFindVirtualHostInRdsResponse() { verify(requestObserver) .onNext( argThat(new DiscoveryRequestMatcher("", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0000"))); verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); verify(configWatcher, never()).onResourceDoesNotExist(anyString()); @@ -906,7 +907,7 @@ public void matchingVirtualHostDoesNotContainRouteAction() { verify(requestObserver) .onNext( argThat(new DiscoveryRequestMatcher("", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0000"))); verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); verify(configWatcher, never()).onResourceDoesNotExist(anyString()); @@ -930,7 +931,7 @@ public void notifyUpdatedResources() { // Client sends an LDS request for the host name (with port) to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management server sends back an LDS response containing a RouteConfiguration for the // requested Listener directly in-line. @@ -955,7 +956,7 @@ public void notifyUpdatedResources() { // Client sends an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0000"))); // Cluster name is resolved and notified to config watcher. ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); @@ -984,7 +985,7 @@ public void notifyUpdatedResources() { // Client sends an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "1", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0001"))); // Updated cluster name is notified to config watcher. configUpdateCaptor = ArgumentCaptor.forClass(null); @@ -1013,12 +1014,12 @@ public void notifyUpdatedResources() { // Client sends an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "2", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0002"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0002"))); // Client sends an (first) RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "some-route-to-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_RDS, ""))); // Management server sends back an RDS response containing the RouteConfiguration // for the requested resource. @@ -1037,7 +1038,7 @@ public void notifyUpdatedResources() { // Client sent an ACK RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "some-route-to-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0000"))); // Updated cluster name is notified to config watcher again. configUpdateCaptor = ArgumentCaptor.forClass(null); @@ -1060,7 +1061,7 @@ public void notifyUpdatedResources() { // Client sent an ACK RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "1", "some-route-to-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0001"))); // Updated cluster name is notified to config watcher again. configUpdateCaptor = ArgumentCaptor.forClass(null); @@ -1096,7 +1097,7 @@ public void waitRdsResponsesForRequestedResource() { // Client sends an LDS request for the host name (with port) to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management sends back an LDS response telling client to do RDS. Rds rdsConfig = @@ -1118,12 +1119,12 @@ public void waitRdsResponsesForRequestedResource() { // Client sends an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0000"))); // Client sends an (first) RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_RDS, ""))); ScheduledTask rdsRespTimer = Iterables.getOnlyElement( @@ -1148,7 +1149,7 @@ public void waitRdsResponsesForRequestedResource() { // Client sent an ACK RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0000"))); // Client waits for future RDS responses silently. verifyNoMoreInteractions(configWatcher); @@ -1175,7 +1176,7 @@ public void waitRdsResponsesForRequestedResource() { // Client sent an ACK RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "1", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0001"))); // Updated cluster name is notified to config watcher. ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); @@ -1198,7 +1199,7 @@ public void routeConfigurationRemovedNotifiedToWatcher() { // Client sends an LDS request for the host name (with port) to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management sends back an LDS response telling client to do RDS. Rds rdsConfig = @@ -1220,12 +1221,12 @@ public void routeConfigurationRemovedNotifiedToWatcher() { // Client sends an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0000"))); // Client sends an (first) RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_RDS, ""))); // Management server sends back an RDS response containing RouteConfiguration requested. List routeConfigs = ImmutableList.of( @@ -1242,7 +1243,7 @@ public void routeConfigurationRemovedNotifiedToWatcher() { // Client sent an ACK RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_RDS, "0000"))); // Resolved cluster name is notified to config watcher. ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); @@ -1260,7 +1261,7 @@ public void routeConfigurationRemovedNotifiedToWatcher() { // Client sent an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "1", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0001"))); verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); } @@ -1295,7 +1296,7 @@ public void updateRdsRequestResourceWhileInitialResourceFetchInProgress() { // Client sends an (first) RDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_RDS, ""))); ScheduledTask rdsRespTimer = Iterables.getOnlyElement( @@ -1324,7 +1325,7 @@ public void updateRdsRequestResourceWhileInitialResourceFetchInProgress() { // Client sent a new RDS request with updated resource name. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "route-bar.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_RDS, ""))); assertThat(rdsRespTimer.isCancelled()).isTrue(); rdsRespTimer = @@ -1361,7 +1362,7 @@ public void cdsResponseWithoutMatchingResource() { // Client sends a CDS request for the only cluster being watched to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); // Management server sends back a CDS response without Cluster for the requested resource. @@ -1375,7 +1376,7 @@ public void cdsResponseWithoutMatchingResource() { // Client sent an ACK CDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); verify(clusterWatcher, never()).onClusterChanged(any(ClusterUpdate.class)); verify(clusterWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); verify(clusterWatcher, never()).onError(any(Status.class)); @@ -1398,7 +1399,7 @@ public void cdsResponseWithMatchingResource() { // Client sends a CDS request for the only cluster being watched to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); ScheduledTask cdsRespTimer = Iterables.getOnlyElement( fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); @@ -1415,7 +1416,7 @@ public void cdsResponseWithMatchingResource() { // Client sent an ACK CDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); assertThat(cdsRespTimer.isCancelled()).isTrue(); ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); @@ -1439,7 +1440,7 @@ public void cdsResponseWithMatchingResource() { // Client sent an ACK CDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "1", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0001"))); verify(clusterWatcher, times(2)).onClusterChanged(clusterUpdateCaptor.capture()); clusterUpdate = clusterUpdateCaptor.getValue(); @@ -1474,7 +1475,7 @@ public void cdsResponseWithUpstreamTlsContext() { // Client sent an ACK CDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); verify(clusterWatcher, times(1)).onClusterChanged(clusterUpdateCaptor.capture()); ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); @@ -1512,7 +1513,7 @@ public void multipleClusterWatchers() { argThat( new DiscoveryRequestMatcher("", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); // Management server sends back a CDS response contains Cluster for only one of @@ -1530,7 +1531,7 @@ public void multipleClusterWatchers() { argThat( new DiscoveryRequestMatcher("0", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); // Two watchers get notification of cluster update for the cluster they are interested in. ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); @@ -1577,7 +1578,7 @@ public void multipleClusterWatchers() { argThat( new DiscoveryRequestMatcher("1", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0001"))); verifyNoMoreInteractions(watcher1, watcher2); // resource has no change ArgumentCaptor clusterUpdateCaptor3 = ArgumentCaptor.forClass(null); @@ -1607,7 +1608,7 @@ public void watchClusterAlreadyBeingWatched() { // Client sends an CDS request to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); // Management server sends back an CDS response with Cluster for the requested @@ -1621,7 +1622,7 @@ public void watchClusterAlreadyBeingWatched() { // Client sent an ACK CDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); @@ -1665,7 +1666,7 @@ public void addRemoveClusterWatchers() { // Client sends an CDS request to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); // Management server sends back a CDS response with Cluster for the requested // cluster. @@ -1678,7 +1679,7 @@ public void addRemoveClusterWatchers() { // Client sent an ACK CDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); @@ -1698,7 +1699,7 @@ public void addRemoveClusterWatchers() { argThat( new DiscoveryRequestMatcher("0", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); // Management server sends back a CDS response with Cluster for all requested cluster. clusters = ImmutableList.of( @@ -1716,7 +1717,7 @@ public void addRemoveClusterWatchers() { argThat( new DiscoveryRequestMatcher("1", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0001"))); verifyNoMoreInteractions(watcher1); // resource has no change ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); @@ -1735,7 +1736,7 @@ public void addRemoveClusterWatchers() { // that cluster. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "1", "cluster-bar.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0001"))); // Management server has nothing to respond. @@ -1746,7 +1747,7 @@ public void addRemoveClusterWatchers() { .onNext( argThat( new DiscoveryRequestMatcher("1", ImmutableList.of(), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0001"))); // Management server sends back a new CDS response. clusters = ImmutableList.of( @@ -1761,7 +1762,7 @@ public void addRemoveClusterWatchers() { .onNext( argThat( new DiscoveryRequestMatcher("2", ImmutableList.of(), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0002"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0002"))); // Cancelled watchers do not receive notification. verifyNoMoreInteractions(watcher1, watcher2); @@ -1774,7 +1775,7 @@ public void addRemoveClusterWatchers() { // A CDS request is sent to indicate subscription of "cluster-foo.googleapis.com" only. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "2", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0002"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0002"))); // Management server sends back a new CDS response for at least newly requested resources // (it is required to do so). @@ -1800,7 +1801,7 @@ public void addRemoveClusterWatchers() { // A CDS request is sent to re-subscribe the cluster again. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "3", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0003"))); + XdsClientImpl.ADS_TYPE_URL_CDS, "0003"))); } @Test @@ -1816,7 +1817,7 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { .onNext( argThat( new DiscoveryRequestMatcher("", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); @@ -1834,7 +1835,7 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { argThat( new DiscoveryRequestMatcher("", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); fakeClock.forwardTime(1, TimeUnit.SECONDS); @@ -1876,7 +1877,7 @@ public void cdsUpdateForClusterBeingRemoved() { verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); // Management server sends back a CDS response containing requested resource. @@ -1922,7 +1923,7 @@ public void edsResponseWithoutMatchingResource() { // Client sends an EDS request for the only cluster being watched to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); // Management server sends back an EDS response without ClusterLoadAssignment for the requested @@ -1951,7 +1952,7 @@ public void edsResponseWithoutMatchingResource() { // Client sent an ACK EDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); verify(endpointWatcher, never()).onEndpointChanged(any(EndpointUpdate.class)); verify(endpointWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); @@ -1974,7 +1975,7 @@ public void edsResponseWithMatchingResource() { // Client sends an EDS request for the only cluster being watched to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); ScheduledTask edsRespTimeoutTask = Iterables.getOnlyElement( fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); @@ -2017,7 +2018,7 @@ public void edsResponseWithMatchingResource() { // Client sent an ACK EDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); ArgumentCaptor endpointUpdateCaptor = ArgumentCaptor.forClass(null); verify(endpointWatcher).onEndpointChanged(endpointUpdateCaptor.capture()); @@ -2050,7 +2051,7 @@ public void edsResponseWithMatchingResource() { // Client sent an ACK EDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "1", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0001"))); verify(endpointWatcher, times(2)).onEndpointChanged(endpointUpdateCaptor.capture()); endpointUpdate = endpointUpdateCaptor.getValue(); @@ -2077,7 +2078,7 @@ public void multipleEndpointWatchers() { argThat( new DiscoveryRequestMatcher("", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); @@ -2105,7 +2106,7 @@ public void multipleEndpointWatchers() { argThat( new DiscoveryRequestMatcher("0", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); // Two watchers get notification of endpoint update for the cluster they are interested in. ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); @@ -2155,7 +2156,7 @@ public void multipleEndpointWatchers() { argThat( new DiscoveryRequestMatcher("1", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0001"))); // The corresponding watcher gets notified. ArgumentCaptor endpointUpdateCaptor3 = ArgumentCaptor.forClass(null); @@ -2186,7 +2187,7 @@ public void watchEndpointsForClusterAlreadyBeingWatched() { // Client sends first EDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); @@ -2211,7 +2212,7 @@ public void watchEndpointsForClusterAlreadyBeingWatched() { // Client sent an ACK EDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); @@ -2264,7 +2265,7 @@ public void addRemoveEndpointWatchers() { // Client sends an EDS request to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server sends back an EDS response with ClusterLoadAssignment for the requested // cluster. @@ -2286,7 +2287,7 @@ public void addRemoveEndpointWatchers() { // Client sent an ACK EDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); @@ -2311,7 +2312,7 @@ public void addRemoveEndpointWatchers() { argThat( new DiscoveryRequestMatcher("0", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); // Management server sends back an EDS response with ClusterLoadAssignment for one of requested // cluster. @@ -2334,7 +2335,7 @@ public void addRemoveEndpointWatchers() { argThat( new DiscoveryRequestMatcher("1", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0001"))); ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); verify(watcher2).onEndpointChanged(endpointUpdateCaptor2.capture()); @@ -2355,7 +2356,7 @@ public void addRemoveEndpointWatchers() { // sent an new EDS request to unsubscribe from that cluster. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "1", "cluster-bar.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0001"))); // Management server should not respond as it had previously sent the requested resource. @@ -2369,7 +2370,7 @@ public void addRemoveEndpointWatchers() { argThat( new DiscoveryRequestMatcher("1", ImmutableList.of(), // empty resources - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0001"))); // All endpoint watchers have been cancelled. @@ -2400,7 +2401,7 @@ public void addRemoveEndpointWatchers() { argThat( new DiscoveryRequestMatcher("2", ImmutableList.of(), // empty resources - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0002"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0002"))); // Cancelled watchers do not receive notification. verifyNoMoreInteractions(watcher1, watcher2); @@ -2417,7 +2418,7 @@ public void addRemoveEndpointWatchers() { // An EDS request is sent to re-subscribe the cluster again. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "2", "cluster-bar.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0002"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0002"))); // Management server sends back an EDS response for re-subscribed resource. clusterLoadAssignments = ImmutableList.of( @@ -2451,7 +2452,7 @@ public void addRemoveEndpointWatchers() { argThat( new DiscoveryRequestMatcher("3", ImmutableList.of("cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0003"))); + XdsClientImpl.ADS_TYPE_URL_EDS, "0003"))); } @Test @@ -2467,7 +2468,7 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { .onNext( argThat( new DiscoveryRequestMatcher("", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); @@ -2485,7 +2486,7 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { argThat( new DiscoveryRequestMatcher("", ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); fakeClock.forwardTime(1, TimeUnit.SECONDS); @@ -2594,7 +2595,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { // Client sends an LDS request for the host name (with port) to management server. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management server closes the RPC stream immediately. responseObserver.onCompleted(); @@ -2614,7 +2615,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { // Client retried by sending an LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management server closes the RPC stream with an error. responseObserver.onError(Status.UNAVAILABLE.asException()); @@ -2634,7 +2635,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { // Client retried again by sending an LDS. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management server responses with a listener for the requested resource. Rds rdsConfig = @@ -2655,12 +2656,12 @@ public void streamClosedAndRetryWhenResolvingConfig() { // Client sent back an ACK LDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + XdsClientImpl.ADS_TYPE_URL_LDS, "0000"))); // Client sent an RDS request based on the received listener. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_RDS, ""))); // Management server encounters an error and closes the stream. responseObserver.onError(Status.UNKNOWN.asException()); @@ -2674,7 +2675,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { requestObserver = requestObservers.poll(); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // RPC stream closed immediately responseObserver.onError(Status.UNKNOWN.asException()); @@ -2691,7 +2692,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { requestObserver = requestObservers.poll(); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management server sends an LDS response. responseObserver.onNext(ldsResponse); @@ -2726,7 +2727,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { requestObserver = requestObservers.poll(); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verifyNoMoreInteractions(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); } @@ -2758,7 +2759,7 @@ public void streamClosedAndRetry() { // Client sent first CDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); // Start watching endpoint information. xdsClient.watchEndpointData("cluster.googleapis.com", endpointWatcher); @@ -2766,7 +2767,7 @@ public void streamClosedAndRetry() { // Client sent first EDS request. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server closes the RPC stream with an error. responseObserver.onError(Status.UNKNOWN.asException()); @@ -2788,13 +2789,13 @@ public void streamClosedAndRetry() { // Retry resumes requests for all wanted resources. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server becomes unreachable. responseObserver.onError(Status.UNAVAILABLE.asException()); @@ -2817,13 +2818,13 @@ public void streamClosedAndRetry() { requestObserver = requestObservers.poll(); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server is still not reachable. responseObserver.onError(Status.UNAVAILABLE.asException()); @@ -2846,13 +2847,13 @@ public void streamClosedAndRetry() { requestObserver = requestObservers.poll(); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server sends back a CDS response. List clusters = ImmutableList.of( @@ -2879,13 +2880,13 @@ public void streamClosedAndRetry() { verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server becomes unreachable again. responseObserver.onError(Status.UNAVAILABLE.asException()); @@ -2907,13 +2908,13 @@ public void streamClosedAndRetry() { requestObserver = requestObservers.poll(); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); verifyNoMoreInteractions(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, backoffPolicy2); @@ -2952,7 +2953,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management server becomes unreachable. responseObserver.onError(Status.UNAVAILABLE.asException()); @@ -2973,10 +2974,10 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); // Management server is still unreachable. responseObserver.onError(Status.UNAVAILABLE.asException()); @@ -2997,13 +2998,13 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server sends back a CDS response. List clusters = ImmutableList.of( @@ -3019,14 +3020,14 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { // Client updates EDS resource subscription immediately. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", ImmutableList.of(), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Become interested in endpoints of another cluster. xdsClient.watchEndpointData("cluster2.googleapis.com", endpointWatcher); // Client updates EDS resource subscription immediately. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster2.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server closes the RPC stream again. responseObserver.onCompleted(); @@ -3040,13 +3041,13 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { requestObserver = requestObservers.poll(); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster2.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Management server becomes unreachable again. responseObserver.onError(Status.UNAVAILABLE.asException()); @@ -3067,13 +3068,13 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver, never()) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver, never()) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster2.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_EDS, ""))); verifyNoMoreInteractions(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, backoffPolicy2); @@ -3139,7 +3140,7 @@ public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { // Client resumed requests and management server sends back LDS resources again. verify(requestObserver).onNext( eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + XdsClientImpl.ADS_TYPE_URL_LDS, ""))); responseObserver.onNext(response); // Client sent an RDS request for resource "route-foo.googleapis.com" (Omitted). @@ -3737,7 +3738,7 @@ public boolean matches(DiscoveryRequest argument) { if (!resourceNames.equals(new HashSet<>(argument.getResourceNamesList()))) { return false; } - return argument.getNode().equals(NODE.toEnvoyProtoNodeV2()); + return argument.getNode().equals(NODE.toEnvoyProtoNode()); } } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java index d2452c86239..6d1aeedaeb8 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java @@ -17,7 +17,7 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponse; +import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponseV2; import static io.grpc.xds.XdsClientTestHelper.buildListener; import static io.grpc.xds.XdsClientTestHelper.buildRouteConfiguration; import static io.grpc.xds.XdsClientTestHelper.buildVirtualHost; @@ -83,7 +83,6 @@ import java.util.Queue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; - import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -331,7 +330,7 @@ public void ldsResponse_nonMatchingFilterChain_notFoundError() { "cluster-baz.googleapis.com")))) .build())))); DiscoveryResponse response = - buildDiscoveryResponse("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); responseObserver.onNext(response); // Client sends an ACK LDS request. @@ -383,7 +382,7 @@ public void ldsResponseWith_listenerAddressPortMismatch() { filterChainInbound ))); DiscoveryResponse response = - buildDiscoveryResponse("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); responseObserver.onNext(response); // Client sends an ACK LDS request. @@ -435,7 +434,7 @@ public void ldsResponseWith_matchingListenerFound() throws InvalidProtocolBuffer filterChainInbound ))); DiscoveryResponse response = - buildDiscoveryResponse("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); responseObserver.onNext(response); // Client sends an ACK LDS request. @@ -510,7 +509,7 @@ public void notifyUpdatedListener() throws InvalidProtocolBufferException { filterChainInbound ))); DiscoveryResponse response = - buildDiscoveryResponse("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); responseObserver.onNext(response); // Client sends an ACK LDS request. @@ -533,7 +532,7 @@ public void notifyUpdatedListener() throws InvalidProtocolBufferException { filterChainNewInbound ))); DiscoveryResponse response1 = - buildDiscoveryResponse("1", listeners1, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); + buildDiscoveryResponseV2("1", listeners1, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); responseObserver.onNext(response1); // Client sends an ACK LDS request. @@ -606,7 +605,7 @@ public void ldsResponse_nonMatchingIpAddress() { filterChainOutbound ))); DiscoveryResponse response = - buildDiscoveryResponse("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); responseObserver.onNext(response); // Client sends an ACK LDS request. @@ -655,7 +654,7 @@ public void ldsResponse_nonMatchingPort() { filterChainOutbound ))); DiscoveryResponse response = - buildDiscoveryResponse("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); responseObserver.onNext(response); // Client sends an ACK LDS request. @@ -706,7 +705,7 @@ public void streamClosedAndRetry() { filterChainInbound ))); DiscoveryResponse response = - buildDiscoveryResponse("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); responseObserver.onNext(response); ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(null); @@ -768,7 +767,7 @@ public void streamClosedAndRetry() { XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); // Management server sends back a LDS response. - response = buildDiscoveryResponse("1", listeners, + response = buildDiscoveryResponseV2("1", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); responseObserver.onNext(response); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java new file mode 100644 index 00000000000..f2ec57c8f2b --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java @@ -0,0 +1,3763 @@ +/* + * Copyright 2019 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.truth.Truth.assertThat; +import static io.grpc.xds.XdsClientTestHelper.buildCluster; +import static io.grpc.xds.XdsClientTestHelper.buildClusterLoadAssignment; +import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryRequestV2; +import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponseV2; +import static io.grpc.xds.XdsClientTestHelper.buildDropOverload; +import static io.grpc.xds.XdsClientTestHelper.buildLbEndpoint; +import static io.grpc.xds.XdsClientTestHelper.buildListener; +import static io.grpc.xds.XdsClientTestHelper.buildLocalityLbEndpoints; +import static io.grpc.xds.XdsClientTestHelper.buildRouteConfiguration; +import static io.grpc.xds.XdsClientTestHelper.buildSecureCluster; +import static io.grpc.xds.XdsClientTestHelper.buildUpstreamTlsContext; +import static io.grpc.xds.XdsClientTestHelper.buildVirtualHost; +import static org.mockito.AdditionalAnswers.delegatesTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.Any; +import com.google.protobuf.BoolValue; +import com.google.protobuf.UInt32Value; +import com.google.protobuf.util.Durations; +import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment.Policy; +import io.envoyproxy.envoy.api.v2.DiscoveryRequest; +import io.envoyproxy.envoy.api.v2.DiscoveryResponse; +import io.envoyproxy.envoy.api.v2.auth.UpstreamTlsContext; +import io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource; +import io.envoyproxy.envoy.api.v2.core.ConfigSource; +import io.envoyproxy.envoy.api.v2.core.HealthStatus; +import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; +import io.envoyproxy.envoy.api.v2.route.RedirectAction; +import io.envoyproxy.envoy.api.v2.route.WeightedCluster; +import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager; +import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.Rds; +import io.envoyproxy.envoy.config.route.v3.QueryParameterMatcher; +import io.envoyproxy.envoy.config.route.v3.Route; +import io.envoyproxy.envoy.config.route.v3.RouteAction; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.config.route.v3.RouteMatch; +import io.envoyproxy.envoy.config.route.v3.VirtualHost; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.SdsSecretConfig; +import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; +import io.envoyproxy.envoy.service.load_stats.v2.LoadReportingServiceGrpc.LoadReportingServiceImplBase; +import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest; +import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsResponse; +import io.grpc.Context; +import io.grpc.Context.CancellationListener; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.Status.Code; +import io.grpc.SynchronizationContext; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.internal.BackoffPolicy; +import io.grpc.internal.FakeClock; +import io.grpc.internal.FakeClock.ScheduledTask; +import io.grpc.internal.FakeClock.TaskFilter; +import io.grpc.stub.StreamObserver; +import io.grpc.testing.GrpcCleanupRule; +import io.grpc.xds.Bootstrapper.ChannelCreds; +import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.EnvoyProtoData.DropOverload; +import io.grpc.xds.EnvoyProtoData.LbEndpoint; +import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.EnvoyProtoData.Node; +import io.grpc.xds.XdsClient.ClusterUpdate; +import io.grpc.xds.XdsClient.ClusterWatcher; +import io.grpc.xds.XdsClient.ConfigUpdate; +import io.grpc.xds.XdsClient.ConfigWatcher; +import io.grpc.xds.XdsClient.EndpointUpdate; +import io.grpc.xds.XdsClient.EndpointWatcher; +import io.grpc.xds.XdsClient.XdsChannel; +import io.grpc.xds.XdsClient.XdsChannelFactory; +import io.grpc.xds.XdsClientImpl.MessagePrinter; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatcher; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +/** + * Tests for {@link XdsClientImpl} with xDS v2 protocol. + */ +@RunWith(JUnit4.class) +public class XdsClientImplTestV2 { + + private static final String TARGET_AUTHORITY = "foo.googleapis.com:8080"; + + private static final Node NODE = Node.newBuilder().build(); + private static final TaskFilter RPC_RETRY_TASK_FILTER = + new TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString().contains(XdsClientImpl.RpcRetryTask.class.getSimpleName()); + } + }; + + private static final TaskFilter LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = + new TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString() + .contains(XdsClientImpl.LdsResourceFetchTimeoutTask.class.getSimpleName()); + } + }; + + private static final TaskFilter RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = + new TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString() + .contains(XdsClientImpl.RdsResourceFetchTimeoutTask.class.getSimpleName()); + } + }; + + private static final TaskFilter CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = + new TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString() + .contains(XdsClientImpl.CdsResourceFetchTimeoutTask.class.getSimpleName()); + } + }; + + private static final TaskFilter EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = + new TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString() + .contains(XdsClientImpl.EdsResourceFetchTimeoutTask.class.getSimpleName()); + } + }; + + @Rule + public final GrpcCleanupRule cleanupRule = new GrpcCleanupRule(); + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); + } + }); + private final FakeClock fakeClock = new FakeClock(); + + private final Queue> responseObservers = new ArrayDeque<>(); + private final Queue> requestObservers = new ArrayDeque<>(); + private final AtomicBoolean adsEnded = new AtomicBoolean(true); + private final Queue loadReportCalls = new ArrayDeque<>(); + private final AtomicBoolean lrsEnded = new AtomicBoolean(true); + + @Mock + private AggregatedDiscoveryServiceImplBase mockedDiscoveryService; + @Mock + private BackoffPolicy.Provider backoffPolicyProvider; + @Mock + private BackoffPolicy backoffPolicy1; + @Mock + private BackoffPolicy backoffPolicy2; + @Mock + private ConfigWatcher configWatcher; + @Mock + private ClusterWatcher clusterWatcher; + @Mock + private EndpointWatcher endpointWatcher; + + private ManagedChannel channel; + private XdsClientImpl xdsClient; + + @Before + public void setUp() throws IOException { + MockitoAnnotations.initMocks(this); + when(backoffPolicyProvider.get()).thenReturn(backoffPolicy1, backoffPolicy2); + when(backoffPolicy1.nextBackoffNanos()).thenReturn(10L, 100L); + when(backoffPolicy2.nextBackoffNanos()).thenReturn(20L, 200L); + + final String serverName = InProcessServerBuilder.generateName(); + AggregatedDiscoveryServiceImplBase adsServiceImpl = new AggregatedDiscoveryServiceImplBase() { + @Override + public StreamObserver streamAggregatedResources( + final StreamObserver responseObserver) { + assertThat(adsEnded.get()).isTrue(); // ensure previous call was ended + adsEnded.set(false); + Context.current().addListener( + new CancellationListener() { + @Override + public void cancelled(Context context) { + adsEnded.set(true); + } + }, MoreExecutors.directExecutor()); + responseObservers.offer(responseObserver); + @SuppressWarnings("unchecked") + StreamObserver requestObserver = mock(StreamObserver.class); + requestObservers.offer(requestObserver); + return requestObserver; + } + }; + mockedDiscoveryService = + mock(AggregatedDiscoveryServiceImplBase.class, delegatesTo(adsServiceImpl)); + + LoadReportingServiceImplBase lrsServiceImpl = new LoadReportingServiceImplBase() { + @Override + public StreamObserver streamLoadStats( + StreamObserver responseObserver) { + assertThat(lrsEnded.get()).isTrue(); + lrsEnded.set(false); + @SuppressWarnings("unchecked") + StreamObserver requestObserver = mock(StreamObserver.class); + final LoadReportCall call = new LoadReportCall(requestObserver, responseObserver); + Context.current().addListener( + new CancellationListener() { + @Override + public void cancelled(Context context) { + lrsEnded.set(true); + } + }, MoreExecutors.directExecutor()); + loadReportCalls.offer(call); + return requestObserver; + } + }; + + cleanupRule.register( + InProcessServerBuilder + .forName(serverName) + .addService(mockedDiscoveryService) + .addService(lrsServiceImpl) + .directExecutor() + .build() + .start()); + channel = + cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); + + List servers = + ImmutableList.of(new ServerInfo(serverName, ImmutableList.of(), null)); + XdsChannelFactory channelFactory = new XdsChannelFactory() { + @Override + XdsChannel createChannel(List servers) { + ServerInfo serverInfo = Iterables.getOnlyElement(servers); + assertThat(serverInfo.getServerUri()).isEqualTo(serverName); + assertThat(serverInfo.getChannelCredentials()).isEmpty(); + return new XdsChannel(channel, false); + } + }; + + xdsClient = + new XdsClientImpl( + TARGET_AUTHORITY, + servers, + channelFactory, + Node.newBuilder().build(), + syncContext, + fakeClock.getScheduledExecutorService(), + backoffPolicyProvider, + fakeClock.getStopwatchSupplier()); + // Only the connection to management server is established, no RPC request is sent until at + // least one watcher is registered. + assertThat(responseObservers).isEmpty(); + assertThat(requestObservers).isEmpty(); + } + + @After + public void tearDown() { + xdsClient.shutdown(); + assertThat(adsEnded.get()).isTrue(); + assertThat(lrsEnded.get()).isTrue(); + assertThat(channel.isShutdown()).isTrue(); + assertThat(fakeClock.getPendingTasks()).isEmpty(); + } + + // Always test the real workflow and integrity of XdsClient: RDS protocol should always followed + // after at least one LDS request-response, from which the RDS resource name comes. CDS and EDS + // can be tested separately as they are used in a standalone way. + + // Discovery responses should follow management server spec and xDS protocol. See + // https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol. + + /** + * Client receives an LDS response that does not contain a Listener for the requested resource. + * The LDS response is ACKed. + * The config watcher is notified with resource unavailable after its response timer expires. + */ + @Test + public void ldsResponseWithoutMatchingResource() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an LDS request for the host name (with port) to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + List listeners = ImmutableList.of( + Any.pack(buildListener("bar.googleapis.com", + Any.pack(HttpConnectionManager.newBuilder() + .setRouteConfig( + buildRouteConfiguration("route-bar.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("bar.googleapis.com"), + "cluster-bar.googleapis.com")))) + .build()))), + Any.pack(buildListener("baz.googleapis.com", + Any.pack(HttpConnectionManager.newBuilder() + .setRouteConfig( + buildRouteConfiguration("route-baz.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("baz.googleapis.com"), + "cluster-baz.googleapis.com")))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + + verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); + verify(configWatcher, never()).onResourceDoesNotExist(TARGET_AUTHORITY); + verify(configWatcher, never()).onError(any(Status.class)); + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); + assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * An LDS response contains the requested listener and an in-lined RouteConfiguration message for + * that listener. But the RouteConfiguration message is invalid as it does not contain any + * VirtualHost with domains matching the requested hostname. + * The LDS response is NACKed, as if the XdsClient has not received this response. + * The config watcher is notified with an error after its response timer expires.. + */ + @Test + public void failToFindVirtualHostInLdsResponseInLineRouteConfig() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an LDS request for the host name (with port) to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + io.envoyproxy.envoy.api.v2.RouteConfiguration routeConfig = + buildRouteConfiguration( + "route.googleapis.com", + ImmutableList.of( + buildVirtualHost(ImmutableList.of("something does not match"), + "some cluster"), + buildVirtualHost(ImmutableList.of("something else does not match"), + "some other cluster"))); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRouteConfig(routeConfig).build())))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an NACK LDS request. + verify(requestObserver) + .onNext( + argThat(new DiscoveryRequestMatcher("", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + + verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); + verify(configWatcher, never()).onResourceDoesNotExist(TARGET_AUTHORITY); + verify(configWatcher, never()).onError(any(Status.class)); + + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); + assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * Client resolves the virtual host config from an LDS response that contains a + * RouteConfiguration message directly in-line for the requested resource. No RDS is needed. + * The LDS response is ACKed. + * The config watcher is notified with an update. + */ + @Test + public void resolveVirtualHostInLdsResponse() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an LDS request for the host name (with port) to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + ScheduledTask ldsRespTimer = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(ldsRespTimer.isCancelled()).isFalse(); + + List listeners = ImmutableList.of( + Any.pack(buildListener("bar.googleapis.com", + Any.pack(HttpConnectionManager.newBuilder() + .setRouteConfig( + buildRouteConfiguration("route-bar.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("bar.googleapis.com"), + "cluster-bar.googleapis.com")))) + .build()))), + Any.pack(buildListener("baz.googleapis.com", + Any.pack(HttpConnectionManager.newBuilder() + .setRouteConfig( + buildRouteConfiguration("route-baz.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("baz.googleapis.com"), + "cluster-baz.googleapis.com")))) + .build()))), + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig( // target route configuration + buildRouteConfiguration("route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost( // matching virtual host + ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com"), + "cluster.googleapis.com"), + buildVirtualHost( + ImmutableList.of("something does not match"), + "some cluster")))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(ldsRespTimer.isCancelled()).isTrue(); + + // Client sends an ACK request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + + ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "cluster.googleapis.com"); + + verifyNoMoreInteractions(requestObserver); + } + + /** + * Client receives an RDS response (after a previous LDS request-response) that does not contain a + * RouteConfiguration for the requested resource while each received RouteConfiguration is valid. + * The RDS response is ACKed. + * After the resource fetch timeout expires, watcher waiting for the resource is notified + * with resource unavailable. + */ + @Test + public void rdsResponseWithoutMatchingResource() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an LDS request for the host name (with port) to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + + // Client sends an (first) RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Management server should only sends RouteConfiguration messages with at least one + // VirtualHost with domains matching requested hostname. Otherwise, it is invalid data. + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "some resource name does not match route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of(TARGET_AUTHORITY), + "whatever cluster")))), + Any.pack( + buildRouteConfiguration( + "some other resource name does not match route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of(TARGET_AUTHORITY), + "some more whatever cluster"))))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + + verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); + verify(configWatcher, never()).onResourceDoesNotExist(anyString()); + verify(configWatcher, never()).onError(any(Status.class)); + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(configWatcher).onResourceDoesNotExist("route-foo.googleapis.com"); + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * Client resolves the virtual host config from an RDS response for the requested resource. The + * RDS response is ACKed. + * The config watcher is notified with an update. + */ + @Test + public void resolveVirtualHostInRdsResponse() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) + + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Management server should only sends RouteConfiguration messages with at least one + // VirtualHost with domains matching requested hostname. Otherwise, it is invalid data. + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", // target route configuration + ImmutableList.of( + buildVirtualHost(ImmutableList.of("something does not match"), + "some cluster"), + buildVirtualHost(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), + "cluster.googleapis.com")))), // matching virtual host + Any.pack( + buildRouteConfiguration( + "some resource name does not match route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost(ImmutableList.of("foo.googleapis.com"), + "some more cluster"))))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + + // Client sent an ACK RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + + ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "cluster.googleapis.com"); + } + + /** + * Client resolves the virtual host config with path matching from an RDS response for the + * requested resource. The RDS response is ACKed. + * The config watcher is notified with an update. + */ + @Test + public void resolveVirtualHostWithPathMatchingInRdsResponse() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) + + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Management server should only sends RouteConfiguration messages with at least one + // VirtualHost with domains matching requested hostname. Otherwise, it is invalid data. + List routeConfigs = + ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", + ImmutableList.of( + io.envoyproxy.envoy.api.v2.route.VirtualHost.newBuilder() + .setName("virtualhost00.googleapis.com") // don't care + // domains wit a match. + .addAllDomains(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com")) + .addRoutes( + io.envoyproxy.envoy.api.v2.route.Route.newBuilder() + // path match with cluster route + .setRoute( + io.envoyproxy.envoy.api.v2.route.RouteAction.newBuilder() + .setCluster("cl1.googleapis.com")) + .setMatch( + io.envoyproxy.envoy.api.v2.route.RouteMatch.newBuilder() + .setPath("/service1/method1"))) + .addRoutes( + io.envoyproxy.envoy.api.v2.route.Route.newBuilder() + // path match with weighted cluster route + .setRoute( + io.envoyproxy.envoy.api.v2.route.RouteAction.newBuilder() + .setWeightedClusters( + WeightedCluster.newBuilder() + .addClusters( + WeightedCluster.ClusterWeight.newBuilder() + .setWeight(UInt32Value.of(30)) + .setName("cl21.googleapis.com")) + .addClusters( + WeightedCluster.ClusterWeight.newBuilder() + .setWeight(UInt32Value.of(70)) + .setName("cl22.googleapis.com")))) + .setMatch( + io.envoyproxy.envoy.api.v2.route.RouteMatch.newBuilder() + .setPath("/service2/method2"))) + .addRoutes( + io.envoyproxy.envoy.api.v2.route.Route.newBuilder() + // prefix match with cluster route + .setRoute( + io.envoyproxy.envoy.api.v2.route.RouteAction.newBuilder() + .setCluster("cl1.googleapis.com")) + .setMatch( + io.envoyproxy.envoy.api.v2.route.RouteMatch.newBuilder() + .setPrefix("/service1/"))) + .addRoutes( + io.envoyproxy.envoy.api.v2.route.Route.newBuilder() + // default match with cluster route + .setRoute( + io.envoyproxy.envoy.api.v2.route.RouteAction.newBuilder() + .setCluster("cluster.googleapis.com")) + .setMatch( + io.envoyproxy.envoy.api.v2.route.RouteMatch.newBuilder() + .setPrefix(""))) + .build())))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + + // Client sent an ACK RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + + ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); + List routes = configUpdateCaptor.getValue().getRoutes(); + assertThat(routes).hasSize(4); + assertThat(routes.get(0)).isEqualTo( + new EnvoyProtoData.Route( + // path match with cluster route + new io.grpc.xds.RouteMatch( + /* prefix= */ null, + /* path= */ "/service1/method1"), + new EnvoyProtoData.RouteAction("cl1.googleapis.com", null))); + assertThat(routes.get(1)).isEqualTo( + new EnvoyProtoData.Route( + // path match with weighted cluster route + new io.grpc.xds.RouteMatch( + /* prefix= */ null, + /* path= */ "/service2/method2"), + new EnvoyProtoData.RouteAction( + null, + ImmutableList.of( + new EnvoyProtoData.ClusterWeight("cl21.googleapis.com", 30), + new EnvoyProtoData.ClusterWeight("cl22.googleapis.com", 70) + )))); + assertThat(routes.get(2)).isEqualTo( + new EnvoyProtoData.Route( + // prefix match with cluster route + new io.grpc.xds.RouteMatch( + /* prefix= */ "/service1/", + /* path= */ null), + new EnvoyProtoData.RouteAction("cl1.googleapis.com", null))); + assertThat(routes.get(3)).isEqualTo( + new EnvoyProtoData.Route( + // default match with cluster route + new io.grpc.xds.RouteMatch( + /* prefix= */ "", + /* path= */ null), + new EnvoyProtoData.RouteAction( + "cluster.googleapis.com", null))); + } + + /** + * Client receives an RDS response (after a previous LDS request-response) containing a + * RouteConfiguration message for the requested resource. But the RouteConfiguration message + * is invalid as it does not contain any VirtualHost with domains matching the requested + * hostname. + * The RDS response is NACKed, as if the XdsClient has not received this response. + */ + @Test + public void failToFindVirtualHostInRdsResponse() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) + + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost(ImmutableList.of("something does not match"), + "some cluster"), + buildVirtualHost( + ImmutableList.of("something else does not match", "also does not match"), + "cluster.googleapis.com")))), + Any.pack( + buildRouteConfiguration( + "some resource name does not match route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost(ImmutableList.of("one more does not match"), + "some more cluster"))))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an NACK RDS request. + verify(requestObserver) + .onNext( + argThat(new DiscoveryRequestMatcher("", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + + verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); + verify(configWatcher, never()).onResourceDoesNotExist(anyString()); + verify(configWatcher, never()).onError(any(Status.class)); + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(configWatcher).onResourceDoesNotExist("route-foo.googleapis.com"); + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * Client receives an RDS response (after a previous LDS request-response) containing a + * RouteConfiguration message for the requested resource. But the RouteConfiguration message + * is invalid as the VirtualHost with domains matching the requested hostname contains invalid + * data, its RouteAction message is absent. + * The RDS response is NACKed, as if the XdsClient has not received this response. + */ + @Test + public void matchingVirtualHostDoesNotContainRouteAction() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) + + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // A VirtualHost with a Route that contains only redirect configuration. + io.envoyproxy.envoy.api.v2.route.VirtualHost virtualHost = + io.envoyproxy.envoy.api.v2.route.VirtualHost.newBuilder() + .setName("virtualhost00.googleapis.com") // don't care + .addDomains(TARGET_AUTHORITY) + .addRoutes( + io.envoyproxy.envoy.api.v2.route.Route.newBuilder() + .setRedirect( + RedirectAction.newBuilder() + .setHostRedirect("bar.googleapis.com") + .setPortRedirect(443))) + .build(); + + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration("route-foo.googleapis.com", + ImmutableList.of(virtualHost)))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an NACK RDS request. + verify(requestObserver) + .onNext( + argThat(new DiscoveryRequestMatcher("", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + + verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); + verify(configWatcher, never()).onResourceDoesNotExist(anyString()); + verify(configWatcher, never()).onError(any(Status.class)); + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(configWatcher).onResourceDoesNotExist("route-foo.googleapis.com"); + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * Client receives LDS/RDS responses for updating resources previously received. + * + *

Tests for streaming behavior. + */ + @Test + public void notifyUpdatedResources() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an LDS request for the host name (with port) to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // Management server sends back an LDS response containing a RouteConfiguration for the + // requested Listener directly in-line. + io.envoyproxy.envoy.api.v2.RouteConfiguration routeConfig = + buildRouteConfiguration( + "route-foo.googleapis.com", // target route configuration + ImmutableList.of( + buildVirtualHost( // matching virtual host + ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), + "cluster.googleapis.com"), + buildVirtualHost(ImmutableList.of("something does not match"), + "some cluster"))); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRouteConfig(routeConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + + // Cluster name is resolved and notified to config watcher. + ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "cluster.googleapis.com"); + + // Management sends back another LDS response containing updates for the requested Listener. + routeConfig = + buildRouteConfiguration( + "another-route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), + "another-cluster.googleapis.com"), + buildVirtualHost(ImmutableList.of("something does not match"), + "some cluster"))); + + listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRouteConfig(routeConfig).build()))) + ); + response = + buildDiscoveryResponseV2("1", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"))); + + // Updated cluster name is notified to config watcher. + configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher, times(2)).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "another-cluster.googleapis.com"); + + // Management server sends back another LDS response containing updates for the requested + // Listener and telling client to do RDS. + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("some-route-to-foo.googleapis.com") + .build(); + + listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + response = + buildDiscoveryResponseV2("2", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0002"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "2", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0002"))); + + // Client sends an (first) RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "some-route-to-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + + // Management server sends back an RDS response containing the RouteConfiguration + // for the requested resource. + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "some-route-to-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost(ImmutableList.of("something does not match"), + "some cluster"), + buildVirtualHost(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), + "some-other-cluster.googleapis.com"))))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "some-route-to-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + + // Updated cluster name is notified to config watcher again. + configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher, times(3)).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "some-other-cluster.googleapis.com"); + + // Management server sends back another RDS response containing updated information for the + // RouteConfiguration currently in-use by client. + routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "some-route-to-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), + "an-updated-cluster.googleapis.com"))))); + response = buildDiscoveryResponseV2( + "1", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "some-route-to-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"))); + + // Updated cluster name is notified to config watcher again. + configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher, times(4)).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "an-updated-cluster.googleapis.com"); + + // Management server sends back an LDS response indicating all Listener resources are removed. + response = + buildDiscoveryResponseV2("3", ImmutableList.of(), + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0003"); + responseObserver.onNext(response); + + verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); + } + + // TODO(chengyuanzhang): tests for timeout waiting for responses for incremental + // protocols (RDS/EDS). + + /** + * Client receives multiple RDS responses without RouteConfiguration for the requested + * resource. It should continue waiting until such an RDS response arrives, as RDS + * protocol is incremental. + * + *

Tests for RDS incremental protocol behavior. + */ + @Test + public void waitRdsResponsesForRequestedResource() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an LDS request for the host name (with port) to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // Management sends back an LDS response telling client to do RDS. + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + + // Client sends an (first) RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + + ScheduledTask rdsRespTimer = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(rdsRespTimer.isCancelled()).isFalse(); + + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 2, TimeUnit.SECONDS); + + // Management server sends back an RDS response that does not contain RouteConfiguration + // for the requested resource. + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "some resource name does not match route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of(TARGET_AUTHORITY), + "some more cluster"))))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + + // Client waits for future RDS responses silently. + verifyNoMoreInteractions(configWatcher); + assertThat(rdsRespTimer.isCancelled()).isFalse(); + + fakeClock.forwardTime(1, TimeUnit.SECONDS); + + // Management server sends back another RDS response containing the RouteConfiguration + // for the requested resource. + routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", // target route configuration + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("something does not match"), + "some cluster"), + buildVirtualHost( // matching virtual host + ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), + "another-cluster.googleapis.com"))))); + response = buildDiscoveryResponseV2( + "1", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"))); + + // Updated cluster name is notified to config watcher. + ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "another-cluster.googleapis.com"); + assertThat(rdsRespTimer.isCancelled()).isTrue(); + } + + /** + * An RouteConfiguration is removed by server by sending client an LDS response removing the + * corresponding Listener. + */ + @Test + public void routeConfigurationRemovedNotifiedToWatcher() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an LDS request for the host name (with port) to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // Management sends back an LDS response telling client to do RDS. + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + + // Client sends an (first) RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + + // Management server sends back an RDS response containing RouteConfiguration requested. + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", // target route configuration + ImmutableList.of( + buildVirtualHost( + ImmutableList.of(TARGET_AUTHORITY), // matching virtual host + "cluster.googleapis.com"))))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); + + // Resolved cluster name is notified to config watcher. + ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "cluster.googleapis.com"); + + // Management server sends back another LDS response with the previous Listener (currently + // in-use by client) removed as the RouteConfiguration it references to is absent. + response = + buildDiscoveryResponseV2("1", ImmutableList.of(), // empty + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"))); + + verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); + } + + /** + * Management server sends another LDS response for updating the RDS resource to be requested + * while client is currently requesting for a previously given RDS resource name. + */ + @Test + public void updateRdsRequestResourceWhileInitialResourceFetchInProgress() { + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Management sends back an LDS response telling client to do RDS. + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sends an (first) RDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + + ScheduledTask rdsRespTimer = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(rdsRespTimer.isCancelled()).isFalse(); + + // Management sends back another LDS response updating the Listener information to use + // another resource name for doing RDS. + rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-bar.googleapis.com") + .build(); + + listeners = ImmutableList.of( + Any.pack( + buildListener( + TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + response = buildDiscoveryResponseV2("1", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent a new RDS request with updated resource name. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-bar.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + + assertThat(rdsRespTimer.isCancelled()).isTrue(); + rdsRespTimer = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(rdsRespTimer.isCancelled()).isFalse(); + + // Management server sends back an RDS response containing RouteConfiguration requested. + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-bar.googleapis.com", // target route configuration + ImmutableList.of( + buildVirtualHost( + ImmutableList.of(TARGET_AUTHORITY), // matching virtual host + "cluster.googleapis.com"))))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(rdsRespTimer.isCancelled()).isTrue(); + } + + /** + * Client receives an CDS response that does not contain a Cluster for the requested resource + * while each received Cluster is valid. The CDS response is ACKed. Cluster watchers are notified + * with resource unavailable after initial resource fetch timeout has expired. + */ + @Test + public void cdsResponseWithoutMatchingResource() { + xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends a CDS request for the only cluster being watched to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Management server sends back a CDS response without Cluster for the requested resource. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-bar.googleapis.com", null, false)), + Any.pack(buildCluster("cluster-baz.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + verify(clusterWatcher, never()).onClusterChanged(any(ClusterUpdate.class)); + verify(clusterWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(clusterWatcher, never()).onError(any(Status.class)); + + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(clusterWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * Normal workflow of receiving a CDS response containing Cluster message for a requested + * cluster. + */ + @Test + public void cdsResponseWithMatchingResource() { + xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends a CDS request for the only cluster being watched to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + ScheduledTask cdsRespTimer = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + + // Management server sends back a CDS response without Cluster for the requested resource. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-bar.googleapis.com", null, false)), + Any.pack(buildCluster("cluster-foo.googleapis.com", null, false)), + Any.pack(buildCluster("cluster-baz.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + assertThat(cdsRespTimer.isCancelled()).isTrue(); + + ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); + verify(clusterWatcher).onClusterChanged(clusterUpdateCaptor.capture()); + ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); + assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate.getEdsServiceName()).isNull(); + assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate.getLrsServerName()).isNull(); + + // Management server sends back another CDS response updating the requested Cluster. + clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-bar.googleapis.com", null, false)), + Any.pack( + buildCluster("cluster-foo.googleapis.com", "eds-cluster-foo.googleapis.com", true)), + Any.pack(buildCluster("cluster-baz.googleapis.com", null, false))); + response = + buildDiscoveryResponseV2("1", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + + verify(clusterWatcher, times(2)).onClusterChanged(clusterUpdateCaptor.capture()); + clusterUpdate = clusterUpdateCaptor.getValue(); + assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate.getEdsServiceName()) + .isEqualTo("eds-cluster-foo.googleapis.com"); + assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate.getLrsServerName()).isEqualTo(""); + } + + /** + * CDS response containing UpstreamTlsContext for a cluster. + */ + @Test + public void cdsResponseWithUpstreamTlsContext() { + xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Management server sends back CDS response with UpstreamTlsContext. + UpstreamTlsContext testUpstreamTlsContext = + buildUpstreamTlsContext("secret1", "unix:/var/uds2"); + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-bar.googleapis.com", null, false)), + Any.pack(buildSecureCluster("cluster-foo.googleapis.com", + "eds-cluster-foo.googleapis.com", true, testUpstreamTlsContext)), + Any.pack(buildCluster("cluster-baz.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); + verify(clusterWatcher, times(1)).onClusterChanged(clusterUpdateCaptor.capture()); + ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); + EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = clusterUpdate + .getUpstreamTlsContext(); + SdsSecretConfig validationContextSdsSecretConfig = upstreamTlsContext.getCommonTlsContext() + .getValidationContextSdsSecretConfig(); + assertThat(validationContextSdsSecretConfig.getName()).isEqualTo("secret1"); + assertThat( + Iterables.getOnlyElement( + validationContextSdsSecretConfig + .getSdsConfig() + .getApiConfigSource() + .getGrpcServicesList()) + .getGoogleGrpc() + .getTargetUri()) + .isEqualTo("unix:/var/uds2"); + } + + @Test + public void multipleClusterWatchers() { + ClusterWatcher watcher1 = mock(ClusterWatcher.class); + ClusterWatcher watcher2 = mock(ClusterWatcher.class); + ClusterWatcher watcher3 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); + xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher3); + + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends a CDS request containing all clusters being watched to management server. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); + + // Management server sends back a CDS response contains Cluster for only one of + // requested cluster. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + // Client sent an ACK CDS request. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("0", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + + // Two watchers get notification of cluster update for the cluster they are interested in. + ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); + ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); + assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate1.getEdsServiceName()).isNull(); + assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate1.getLrsServerName()).isNull(); + + ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); + ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); + assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate2.getEdsServiceName()).isNull(); + assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate2.getLrsServerName()).isNull(); + + verify(watcher3, never()).onClusterChanged(any(ClusterUpdate.class)); + verify(watcher3, never()).onResourceDoesNotExist("cluster-bar.googleapis.com"); + verify(watcher3, never()).onError(any(Status.class)); + + // The other watcher gets an error notification for cluster not found after its timer expired. + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(watcher3).onResourceDoesNotExist("cluster-bar.googleapis.com"); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + + // Management server sends back another CDS response contains Clusters for all + // requested clusters. + clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", null, false)), + Any.pack( + buildCluster("cluster-bar.googleapis.com", + "eds-cluster-bar.googleapis.com", true))); + response = buildDiscoveryResponseV2("1", clusters, + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("1", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + + verifyNoMoreInteractions(watcher1, watcher2); // resource has no change + ArgumentCaptor clusterUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onClusterChanged(clusterUpdateCaptor3.capture()); + ClusterUpdate clusterUpdate3 = clusterUpdateCaptor3.getValue(); + assertThat(clusterUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(clusterUpdate3.getEdsServiceName()) + .isEqualTo("eds-cluster-bar.googleapis.com"); + assertThat(clusterUpdate3.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate3.getLrsServerName()).isEqualTo(""); + } + + /** + * (CDS response caching behavior) Adding cluster watchers interested in some cluster that + * some other endpoint watcher had already been watching on will result in cluster update + * notified to the newly added watcher immediately, without sending new CDS requests. + */ + @Test + public void watchClusterAlreadyBeingWatched() { + ClusterWatcher watcher1 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + + // Streaming RPC starts after a first watcher is added. + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an CDS request to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Management server sends back an CDS response with Cluster for the requested + // cluster. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + + ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); + ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); + assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate1.getEdsServiceName()).isNull(); + assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate1.getLrsServerName()).isNull(); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + + // Another cluster watcher interested in the same cluster is added. + ClusterWatcher watcher2 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); + + // Since the client has received cluster update for this cluster before, cached result is + // notified to the newly added watcher immediately. + ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); + ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); + assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate2.getEdsServiceName()).isNull(); + assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate2.getLrsServerName()).isNull(); + + verifyNoMoreInteractions(requestObserver); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * Basic operations of adding/canceling cluster data watchers. + */ + @Test + public void addRemoveClusterWatchers() { + ClusterWatcher watcher1 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + + // Streaming RPC starts after a first watcher is added. + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an CDS request to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + + // Management server sends back a CDS response with Cluster for the requested + // cluster. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + + ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); + ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); + assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate1.getEdsServiceName()).isNull(); + assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate1.getLrsServerName()).isNull(); + + // Add another cluster watcher for a different cluster. + ClusterWatcher watcher2 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher2); + + // Client sent a new CDS request for all interested resources. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("0", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); + + // Management server sends back a CDS response with Cluster for all requested cluster. + clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", null, false)), + Any.pack( + buildCluster("cluster-bar.googleapis.com", + "eds-cluster-bar.googleapis.com", true))); + response = buildDiscoveryResponseV2("1", clusters, + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request for all interested resources. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("1", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + verifyNoMoreInteractions(watcher1); // resource has no change + ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); + ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); + assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(clusterUpdate2.getEdsServiceName()) + .isEqualTo("eds-cluster-bar.googleapis.com"); + assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate2.getLrsServerName()).isEqualTo(""); + + // Cancel one of the watcher. + xdsClient.cancelClusterDataWatch("cluster-foo.googleapis.com", watcher1); + + // Since the cancelled watcher was the last watcher interested in that cluster (but there + // is still interested resource), client sent an new CDS request to unsubscribe from + // that cluster. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-bar.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + + // Management server has nothing to respond. + + // Cancel the other watcher. All resources have been unsubscribed. + xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher2); + + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("1", ImmutableList.of(), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); + + // Management server sends back a new CDS response. + clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", null, true)), + Any.pack( + buildCluster("cluster-bar.googleapis.com", null, false))); + response = + buildDiscoveryResponseV2("2", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0002"); + responseObserver.onNext(response); + + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("2", ImmutableList.of(), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0002"))); + + // Cancelled watchers do not receive notification. + verifyNoMoreInteractions(watcher1, watcher2); + + // A new cluster watcher is added to watch cluster foo again. + ClusterWatcher watcher3 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher3); + verify(watcher3, never()).onClusterChanged(any(ClusterUpdate.class)); + + // A CDS request is sent to indicate subscription of "cluster-foo.googleapis.com" only. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "2", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0002"))); + + // Management server sends back a new CDS response for at least newly requested resources + // (it is required to do so). + clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", null, true)), + Any.pack( + buildCluster("cluster-bar.googleapis.com", null, false))); + response = + buildDiscoveryResponseV2("3", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0003"); + responseObserver.onNext(response); + + // Notified with cached data immediately. + ArgumentCaptor clusterUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onClusterChanged(clusterUpdateCaptor3.capture()); + ClusterUpdate clusterUpdate3 = clusterUpdateCaptor3.getValue(); + assertThat(clusterUpdate3.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate3.getEdsServiceName()).isNull(); + assertThat(clusterUpdate3.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate2.getLrsServerName()).isEqualTo(""); + + verifyNoMoreInteractions(watcher1, watcher2); + + // A CDS request is sent to re-subscribe the cluster again. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "3", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0003"))); + } + + @Test + public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { + ClusterWatcher watcher1 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + + // Streaming RPC starts after a first watcher is added. + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an EDS request to management server. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); + + ClusterWatcher watcher2 = mock(ClusterWatcher.class); + ClusterWatcher watcher3 = mock(ClusterWatcher.class); + ClusterWatcher watcher4 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); + xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher3); + xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher4); + + // Client sends a new CDS request for updating the latest resource subscription. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); + + fakeClock.forwardTime(1, TimeUnit.SECONDS); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // CDS resource "cluster-foo.googleapis.com" is known to be absent. + verify(watcher1).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(watcher2).onResourceDoesNotExist("cluster-foo.googleapis.com"); + + // The absence result is known immediately. + ClusterWatcher watcher5 = mock(ClusterWatcher.class); + xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher5); + verify(watcher5).onResourceDoesNotExist("cluster-foo.googleapis.com"); + + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + ScheduledTask timeoutTask = Iterables.getOnlyElement(fakeClock.getPendingTasks()); + + // Cancel watchers while discovery for resource "cluster-bar.googleapis.com" is still + // in progress. + xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher3); + assertThat(timeoutTask.isCancelled()).isFalse(); + xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher4); + + // Client sends a CDS request for resource subscription update (Omitted). + + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + assertThat(timeoutTask.isCancelled()).isTrue(); + + verifyZeroInteractions(watcher3, watcher4); + } + + @Test + public void cdsUpdateForClusterBeingRemoved() { + xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Management server sends back a CDS response containing requested resource. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", null, true))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK CDS request (Omitted). + + ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); + verify(clusterWatcher).onClusterChanged(clusterUpdateCaptor.capture()); + ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); + assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(clusterUpdate.getEdsServiceName()).isNull(); + assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(clusterUpdate.getLrsServerName()).isEqualTo(""); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + + // No cluster is available. + response = + buildDiscoveryResponseV2("1", ImmutableList.of(), + XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); + responseObserver.onNext(response); + + verify(clusterWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + } + + /** + * Client receives an EDS response that does not contain a ClusterLoadAssignment for the + * requested resource while each received ClusterLoadAssignment is valid. + * The EDS response is ACKed. + * After the resource fetch timeout expires, watchers waiting for the resource is notified + * with resource unavailable. + */ + @Test + public void edsResponseWithoutMatchingResource() { + xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an EDS request for the only cluster being watched to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Management server sends back an EDS response without ClusterLoadAssignment for the requested + // cluster. + List clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-bar.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0)), + ImmutableList.of())), + Any.pack(buildClusterLoadAssignment("cluster-baz.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region2", "zone2", "subzone2", + ImmutableList.of( + buildLbEndpoint("192.168.234.52", 8888, HealthStatus.UNKNOWN, 5)), + 6, 1)), + ImmutableList.of()))); + + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + + verify(endpointWatcher, never()).onEndpointChanged(any(EndpointUpdate.class)); + verify(endpointWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(endpointWatcher, never()).onError(any(Status.class)); + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(endpointWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * Normal workflow of receiving an EDS response containing ClusterLoadAssignment message for + * a requested cluster. + */ + @Test + public void edsResponseWithMatchingResource() { + xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an EDS request for the only cluster being watched to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + ScheduledTask edsRespTimeoutTask = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(edsRespTimeoutTask.isCancelled()).isFalse(); + + // Management server sends back an EDS response with ClusterLoadAssignment for the requested + // cluster. + List clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0), + buildLocalityLbEndpoints("region3", "zone3", "subzone3", + ImmutableList.of(), + 2, 1), /* locality with 0 endpoint */ + buildLocalityLbEndpoints("region4", "zone4", "subzone4", + ImmutableList.of( + buildLbEndpoint("192.168.142.5", 80, HealthStatus.UNKNOWN, 5)), + 0, 2) /* locality with 0 weight */), + ImmutableList.of( + buildDropOverload("lb", 200), + buildDropOverload("throttle", 1000)))), + Any.pack(buildClusterLoadAssignment("cluster-baz.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region2", "zone2", "subzone2", + ImmutableList.of( + buildLbEndpoint("192.168.234.52", 8888, HealthStatus.UNKNOWN, 5)), + 6, 1)), + ImmutableList.of()))); + + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(edsRespTimeoutTask.isCancelled()).isTrue(); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + + ArgumentCaptor endpointUpdateCaptor = ArgumentCaptor.forClass(null); + verify(endpointWatcher).onEndpointChanged(endpointUpdateCaptor.capture()); + EndpointUpdate endpointUpdate = endpointUpdateCaptor.getValue(); + assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(endpointUpdate.getDropPolicies()) + .containsExactly( + new DropOverload("lb", 200), + new DropOverload("throttle", 1000)); + assertThat(endpointUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, + 2, true)), 1, 0), + new Locality("region3", "zone3", "subzone3"), + new LocalityLbEndpoints(ImmutableList.of(), 2, 1)); + + clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo.googleapis.com", + // 0 locality + ImmutableList.of(), + ImmutableList.of()))); + response = + buildDiscoveryResponseV2( + "1", clusterLoadAssignments, XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + + verify(endpointWatcher, times(2)).onEndpointChanged(endpointUpdateCaptor.capture()); + endpointUpdate = endpointUpdateCaptor.getValue(); + assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(endpointUpdate.getDropPolicies()).isEmpty(); + assertThat(endpointUpdate.getLocalityLbEndpointsMap()).isEmpty(); + } + + @Test + public void multipleEndpointWatchers() { + EndpointWatcher watcher1 = mock(EndpointWatcher.class); + EndpointWatcher watcher2 = mock(EndpointWatcher.class); + EndpointWatcher watcher3 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); + xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); + + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an EDS request containing all clusters being watched to management server. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); + + // Management server sends back an EDS response contains ClusterLoadAssignment for only one of + // requested cluster. + List clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0)), + ImmutableList.of()))); + + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("0", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + + // Two watchers get notification of endpoint update for the cluster they are interested in. + ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); + EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); + assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, + 2, true)), 1, 0)); + + ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher1).onEndpointChanged(endpointUpdateCaptor2.capture()); + EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); + assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, + 2, true)), 1, 0)); + + verifyZeroInteractions(watcher3); + + // Management server sends back another EDS response contains ClusterLoadAssignment for the + // other requested cluster. + clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-bar.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region2", "zone2", "subzone2", + ImmutableList.of( + buildLbEndpoint("192.168.234.52", 8888, HealthStatus.UNKNOWN, 5)), + 6, 0)), + ImmutableList.of()))); + + response = buildDiscoveryResponseV2("1", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("1", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + + // The corresponding watcher gets notified. + ArgumentCaptor endpointUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onEndpointChanged(endpointUpdateCaptor3.capture()); + EndpointUpdate endpointUpdate3 = endpointUpdateCaptor3.getValue(); + assertThat(endpointUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(endpointUpdate3.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region2", "zone2", "subzone2"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.234.52", 8888, + 5, true)), 6, 0)); + } + + /** + * (EDS response caching behavior) An endpoint watcher is registered for a cluster that already + * has some other endpoint watchers watching on. Endpoint information received previously is + * in local cache and notified to the new watcher immediately. + */ + @Test + public void watchEndpointsForClusterAlreadyBeingWatched() { + EndpointWatcher watcher1 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends first EDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // Management server sends back an EDS response containing ClusterLoadAssignments for + // some cluster not requested. + List clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0)), + ImmutableList.of()))); + + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + + ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); + EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); + assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(endpointUpdate1.getDropPolicies()).isEmpty(); + assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, + 2, true)), 1, 0)); + + // A second endpoint watcher is registered for endpoints in the same cluster. + EndpointWatcher watcher2 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); + + // Cached endpoint information is notified to the new watcher immediately, without sending + // another EDS request. + ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onEndpointChanged(endpointUpdateCaptor2.capture()); + EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); + assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(endpointUpdate2.getDropPolicies()).isEmpty(); + assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, + 2, true)), 1, 0)); + + verifyNoMoreInteractions(requestObserver); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * Basic operations of adding/canceling endpoint data watchers. + */ + @Test + public void addRemoveEndpointWatchers() { + EndpointWatcher watcher1 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + + // Streaming RPC starts after a first watcher is added. + StreamObserver responseObserver = responseObservers.poll(); + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an EDS request to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server sends back an EDS response with ClusterLoadAssignment for the requested + // cluster. + List clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2), + buildLbEndpoint("192.132.53.5", 80, HealthStatus.UNHEALTHY, 5)), + 1, 0)), + ImmutableList.of()))); + + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + + ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); + EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); + assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, 2, true), + new LbEndpoint("192.132.53.5", 80,5, false)), + 1, 0)); + + // Add another endpoint watcher for a different cluster. + EndpointWatcher watcher2 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher2); + + // Client sent a new EDS request for all interested resources. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("0", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); + + // Management server sends back an EDS response with ClusterLoadAssignment for one of requested + // cluster. + clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-bar.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region2", "zone2", "subzone2", + ImmutableList.of( + buildLbEndpoint("192.168.312.6", 443, HealthStatus.HEALTHY, 1)), + 6, 0)), + ImmutableList.of()))); + + response = buildDiscoveryResponseV2("1", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"); + responseObserver.onNext(response); + + // Client sent an ACK EDS request for all interested resources. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("1", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + + ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onEndpointChanged(endpointUpdateCaptor2.capture()); + EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); + assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region2", "zone2", "subzone2"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.312.6", 443, 1, true)), + 6, 0)); + + // Cancel one of the watcher. + xdsClient.cancelEndpointDataWatch("cluster-foo.googleapis.com", watcher1); + + // Since the cancelled watcher was the last watcher interested in that cluster, client + // sent an new EDS request to unsubscribe from that cluster. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-bar.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + + // Management server should not respond as it had previously sent the requested resource. + + // Cancel the other watcher. + xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher2); + + // Since the cancelled watcher was the last watcher interested in that cluster, client + // sent an new EDS request to unsubscribe from that cluster. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("1", + ImmutableList.of(), // empty resources + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); + + // All endpoint watchers have been cancelled. + + // Management server sends back an EDS response for updating previously sent resources. + clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region3", "zone3", "subzone3", + ImmutableList.of( + buildLbEndpoint("192.168.432.6", 80, HealthStatus.HEALTHY, 2)), + 3, 0)), + ImmutableList.of())), + Any.pack(buildClusterLoadAssignment("cluster-bar.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region4", "zone4", "subzone4", + ImmutableList.of( + buildLbEndpoint("192.168.75.6", 8888, HealthStatus.HEALTHY, 2)), + 3, 0)), + ImmutableList.of()))); + + response = buildDiscoveryResponseV2("2", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0002"); + responseObserver.onNext(response); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("2", + ImmutableList.of(), // empty resources + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0002"))); + + // Cancelled watchers do not receive notification. + verifyNoMoreInteractions(watcher1, watcher2); + + // A new endpoint watcher is added to watch an old but was no longer interested in cluster. + EndpointWatcher watcher3 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); + + // Nothing should be notified to the new watcher as we are still waiting management server's + // latest response. + // Cached endpoint data should have been purged. + verify(watcher3, never()).onEndpointChanged(any(EndpointUpdate.class)); + + // An EDS request is sent to re-subscribe the cluster again. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "2", "cluster-bar.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0002"))); + + // Management server sends back an EDS response for re-subscribed resource. + clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-bar.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region4", "zone4", "subzone4", + ImmutableList.of( + buildLbEndpoint("192.168.75.6", 8888, HealthStatus.HEALTHY, 2)), + 3, 0)), + ImmutableList.of()))); + + response = buildDiscoveryResponseV2("3", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0003"); + responseObserver.onNext(response); + + ArgumentCaptor endpointUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onEndpointChanged(endpointUpdateCaptor3.capture()); + EndpointUpdate endpointUpdate3 = endpointUpdateCaptor3.getValue(); + assertThat(endpointUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(endpointUpdate3.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region4", "zone4", "subzone4"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.75.6", 8888, 2, true)), + 3, 0)); + + // Client sent an ACK EDS request. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("3", + ImmutableList.of("cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0003"))); + } + + @Test + public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { + EndpointWatcher watcher1 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + + // Streaming RPC starts after a first watcher is added. + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an EDS request to management server. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("", "cluster-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); + + EndpointWatcher watcher2 = mock(EndpointWatcher.class); + EndpointWatcher watcher3 = mock(EndpointWatcher.class); + EndpointWatcher watcher4 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); + xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); + xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher4); + + // Client sends a new EDS request for updating the latest resource subscription. + verify(requestObserver) + .onNext( + argThat( + new DiscoveryRequestMatcher("", + ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); + + fakeClock.forwardTime(1, TimeUnit.SECONDS); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + + // EDS resource "cluster-foo.googleapis.com" is known to be absent. + verify(watcher1).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(watcher2).onResourceDoesNotExist("cluster-foo.googleapis.com"); + + // The absence result is known immediately. + EndpointWatcher watcher5 = mock(EndpointWatcher.class); + xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher5); + verify(watcher5).onResourceDoesNotExist("cluster-foo.googleapis.com"); + + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + ScheduledTask timeoutTask = Iterables.getOnlyElement(fakeClock.getPendingTasks()); + + // Cancel watchers while discovery for resource "cluster-bar.googleapis.com" is still + // in progress. + xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher3); + assertThat(timeoutTask.isCancelled()).isFalse(); + xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher4); + + // Client sends an EDS request for resource subscription update (Omitted). + + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + assertThat(timeoutTask.isCancelled()).isTrue(); + + verifyZeroInteractions(watcher3, watcher4); + } + + @Test + public void cdsUpdateForEdsServiceNameChange() { + xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + StreamObserver responseObserver = responseObservers.poll(); + + // Management server sends back a CDS response containing requested resource. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", "cluster-foo:service-bar", false))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(response); + + xdsClient.watchEndpointData("cluster-foo:service-bar", endpointWatcher); + + // Management server sends back an EDS response for resource "cluster-foo:service-bar". + List clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo:service-bar", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0)), + ImmutableList.of()))); + response = + buildDiscoveryResponseV2("0", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); + responseObserver.onNext(response); + + ArgumentCaptor endpointUpdateCaptor = ArgumentCaptor.forClass(null); + verify(endpointWatcher).onEndpointChanged(endpointUpdateCaptor.capture()); + EndpointUpdate endpointUpdate = endpointUpdateCaptor.getValue(); + assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo:service-bar"); + assertThat(endpointUpdate.getDropPolicies()).isEmpty(); + assertThat(endpointUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, + 2, true)), 1, 0)); + + // Management server sends another CDS response for removing cluster service + // "cluster-foo:service-blade" with replacement of "cluster-foo:service-blade". + clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-foo.googleapis.com", "cluster-foo:service-blade", false))); + response = + buildDiscoveryResponseV2("1", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); + responseObserver.onNext(response); + + // Watcher get notification for endpoint resource "cluster-foo:service-bar" being deleted. + verify(endpointWatcher).onResourceDoesNotExist("cluster-foo:service-bar"); + } + + /** + * RPC stream closed and retry during the period of first time resolving service config + * (LDS/RDS only). + */ + @Test + public void streamClosedAndRetryWhenResolvingConfig() { + InOrder inOrder = + Mockito.inOrder(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, + backoffPolicy2); + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + + ArgumentCaptor> responseObserverCaptor = + ArgumentCaptor.forClass(null); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + StreamObserver responseObserver = + responseObserverCaptor.getValue(); // same as responseObservers.poll() + StreamObserver requestObserver = requestObservers.poll(); + + // Client sends an LDS request for the host name (with port) to management server. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // Management server closes the RPC stream immediately. + responseObserver.onCompleted(); + inOrder.verify(backoffPolicyProvider).get(); + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Retry after backoff. + fakeClock.forwardNanos(9L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + + // Client retried by sending an LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // Management server closes the RPC stream with an error. + responseObserver.onError(Status.UNAVAILABLE.asException()); + verifyNoMoreInteractions(backoffPolicyProvider); + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Retry after backoff. + fakeClock.forwardNanos(99L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + + // Client retried again by sending an LDS. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // Management server responses with a listener for the requested resource. + Rds rdsConfig = + Rds.newBuilder() + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse ldsResponse = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(ldsResponse); + + // Client sent back an ACK LDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); + + // Client sent an RDS request based on the received listener. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); + + // Management server encounters an error and closes the stream. + responseObserver.onError(Status.UNKNOWN.asException()); + + // Reset backoff and retry immediately. + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // RPC stream closed immediately + responseObserver.onError(Status.UNKNOWN.asException()); + inOrder.verify(backoffPolicy2).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Retry after backoff. + fakeClock.forwardNanos(19L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // Management server sends an LDS response. + responseObserver.onNext(ldsResponse); + + // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) + + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", // target route configuration + ImmutableList.of( + buildVirtualHost( + ImmutableList.of(TARGET_AUTHORITY), // matching virtual host + "cluster.googleapis.com"))))); + DiscoveryResponse rdsResponse = + buildDiscoveryResponseV2("0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + // Management server sends an RDS response. + responseObserver.onNext(rdsResponse); + + // Client has resolved the cluster based on the RDS response. + ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); + verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); + assertConfigUpdateContainsSingleClusterRoute( + configUpdateCaptor.getValue(), "cluster.googleapis.com"); + + // RPC stream closed with an error again. + responseObserver.onError(Status.UNKNOWN.asException()); + + // Reset backoff and retry immediately. + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + requestObserver = requestObservers.poll(); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + verifyNoMoreInteractions(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); + } + + /** + * RPC stream close and retry while there are config/cluster/endpoint watchers registered. + */ + @Test + public void streamClosedAndRetry() { + InOrder inOrder = + Mockito.inOrder(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, + backoffPolicy2); + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + + ArgumentCaptor> responseObserverCaptor = + ArgumentCaptor.forClass(null); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + StreamObserver responseObserver = + responseObserverCaptor.getValue(); // same as responseObservers.poll() + StreamObserver requestObserver = requestObservers.poll(); + + waitUntilConfigResolved(responseObserver); + ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(null); + + // Start watching cluster information. + xdsClient.watchClusterData("cluster.googleapis.com", clusterWatcher); + + // Client sent first CDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + + // Start watching endpoint information. + xdsClient.watchEndpointData("cluster.googleapis.com", endpointWatcher); + + // Client sent first EDS request. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server closes the RPC stream with an error. + responseObserver.onError(Status.UNKNOWN.asException()); + verify(configWatcher).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); + verify(clusterWatcher).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); + verify(endpointWatcher).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); + + // Resets backoff and retry immediately. + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + + // Retry resumes requests for all wanted resources. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server becomes unreachable. + responseObserver.onError(Status.UNAVAILABLE.asException()); + verify(configWatcher, times(2)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(clusterWatcher, times(2)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(endpointWatcher, times(2)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Retry after backoff. + fakeClock.forwardNanos(9L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server is still not reachable. + responseObserver.onError(Status.UNAVAILABLE.asException()); + verify(configWatcher, times(3)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(clusterWatcher, times(3)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(endpointWatcher, times(3)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Retry after backoff. + fakeClock.forwardNanos(99L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server sends back a CDS response. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster.googleapis.com", null, false))); + DiscoveryResponse cdsResponse = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(cdsResponse); + + // Client sent an CDS ACK request (Omitted). + + // Management server closes the RPC stream. + responseObserver.onCompleted(); + verify(configWatcher, times(4)).onError(any(Status.class)); + verify(clusterWatcher, times(4)).onError(any(Status.class)); + verify(endpointWatcher, times(4)).onError(any(Status.class)); + + // Resets backoff and retry immediately + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server becomes unreachable again. + responseObserver.onError(Status.UNAVAILABLE.asException()); + verify(configWatcher, times(5)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(clusterWatcher, times(5)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(endpointWatcher, times(5)).onError(statusCaptor.capture()); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + inOrder.verify(backoffPolicy2).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Retry after backoff. + fakeClock.forwardNanos(19L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + requestObserver = requestObservers.poll(); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + verifyNoMoreInteractions(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, + backoffPolicy2); + } + + /** + * RPC stream closed and retry while some cluster/endpoint watchers have changed (added/removed). + */ + @Test + public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { + InOrder inOrder = + Mockito.inOrder(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, + backoffPolicy2); + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + + ArgumentCaptor> responseObserverCaptor = + ArgumentCaptor.forClass(null); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + StreamObserver responseObserver = + responseObserverCaptor.getValue(); // same as responseObservers.poll() + requestObservers.poll(); + + waitUntilConfigResolved(responseObserver); + + // Management server closes RPC stream. + responseObserver.onCompleted(); + + // Resets backoff and retry immediately. + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + StreamObserver requestObserver = requestObservers.poll(); + + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + + // Management server becomes unreachable. + responseObserver.onError(Status.UNAVAILABLE.asException()); + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Start watching cluster information while RPC stream is still in retry backoff. + xdsClient.watchClusterData("cluster.googleapis.com", clusterWatcher); + + // Retry after backoff. + fakeClock.forwardNanos(9L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + + // Management server is still unreachable. + responseObserver.onError(Status.UNAVAILABLE.asException()); + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Start watching endpoint information while RPC stream is still in retry backoff. + xdsClient.watchEndpointData("cluster.googleapis.com", endpointWatcher); + + // Retry after backoff. + fakeClock.forwardNanos(99L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server sends back a CDS response. + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster.googleapis.com", null, false))); + DiscoveryResponse cdsResponse = + buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); + responseObserver.onNext(cdsResponse); + + // Client sent an CDS ACK request (Omitted). + + // No longer interested in endpoint information after RPC resumes. + xdsClient.cancelEndpointDataWatch("cluster.googleapis.com", endpointWatcher); + // Client updates EDS resource subscription immediately. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", ImmutableList.of(), + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Become interested in endpoints of another cluster. + xdsClient.watchEndpointData("cluster2.googleapis.com", endpointWatcher); + // Client updates EDS resource subscription immediately. + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server closes the RPC stream again. + responseObserver.onCompleted(); + + // Resets backoff and retry immediately. + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + requestObserver = requestObservers.poll(); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + // Management server becomes unreachable again. + responseObserver.onError(Status.UNAVAILABLE.asException()); + inOrder.verify(backoffPolicy2).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // No longer interested in previous cluster and endpoints in that cluster. + xdsClient.cancelClusterDataWatch("cluster.googleapis.com", clusterWatcher); + xdsClient.cancelEndpointDataWatch("cluster2.googleapis.com", endpointWatcher); + + // Retry after backoff. + fakeClock.forwardNanos(19L); + assertThat(requestObservers).isEmpty(); + fakeClock.forwardNanos(1L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + requestObserver = requestObservers.poll(); + + verify(requestObserver) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + verify(requestObserver, never()) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); + verify(requestObserver, never()) + .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", + XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); + + verifyNoMoreInteractions(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, + backoffPolicy2); + } + + @Test + public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { + InOrder inOrder = + Mockito.inOrder(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, + backoffPolicy2); + xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); + + ArgumentCaptor> responseObserverCaptor = + ArgumentCaptor.forClass(null); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + StreamObserver responseObserver = + responseObserverCaptor.getValue(); // same as responseObservers.poll() + + // Management server sends back an LDS response telling client to do RDS. + Rds rdsConfig = + Rds.newBuilder() + // Must set to use ADS. + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(response); + + // Client sent an RDS request for resource "route-foo.googleapis.com" (Omitted). + + ScheduledTask rdsRespTimer = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(rdsRespTimer.isCancelled()).isFalse(); + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); + + // RPC stream is broken while the initial fetch for the resource is not complete. + responseObserver.onError(Status.UNAVAILABLE.asException()); + assertThat(rdsRespTimer.isCancelled()).isTrue(); + + // Reset backoff and retry immediately. + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + StreamObserver requestObserver = requestObservers.poll(); + + ScheduledTask ldsRespTimer = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(ldsRespTimer.getDelay(TimeUnit.SECONDS)) + .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); + + // Client resumed requests and management server sends back LDS resources again. + verify(requestObserver).onNext( + eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); + responseObserver.onNext(response); + + // Client sent an RDS request for resource "route-foo.googleapis.com" (Omitted). + + assertThat(ldsRespTimer.isCancelled()).isTrue(); + rdsRespTimer = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(rdsRespTimer.getDelay(TimeUnit.SECONDS)) + .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); + + // Management server sends back an RDS response containing the RouteConfiguration + // for the requested resource. + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", // target route configuration + ImmutableList.of( + buildVirtualHost( + ImmutableList.of(TARGET_AUTHORITY), // matching virtual host + "cluster-foo.googleapis.com"))))); + response = buildDiscoveryResponseV2( + "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(response); + + assertThat(rdsRespTimer.isCancelled()).isTrue(); + + // Resets RPC stream again. + responseObserver.onError(Status.UNAVAILABLE.asException()); + // Reset backoff and retry immediately. + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + + // Client/server resumed LDS/RDS request/response (Omitted). + + // Start watching cluster data. + xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + ScheduledTask cdsRespTimeoutTask = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(cdsRespTimeoutTask.isCancelled()).isFalse(); + fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); + + // RPC stream is broken while the initial fetch for the resource is not complete. + responseObserver.onError(Status.UNAVAILABLE.asException()); + assertThat(cdsRespTimeoutTask.isCancelled()).isTrue(); + inOrder.verify(backoffPolicy2).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + // Retry after backoff. + fakeClock.forwardNanos(20L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + responseObserver = responseObserverCaptor.getValue(); + + // Timer is rescheduled as the client restarts the resource fetch. + cdsRespTimeoutTask = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(cdsRespTimeoutTask.isCancelled()).isFalse(); + assertThat(cdsRespTimeoutTask.getDelay(TimeUnit.SECONDS)) + .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); + + // Start watching endpoint data. + xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + ScheduledTask edsTimeoutTask = + Iterables.getOnlyElement( + fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + assertThat(edsTimeoutTask.getDelay(TimeUnit.SECONDS)) + .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); + + // RPC stream is broken again. + responseObserver.onError(Status.UNAVAILABLE.asException()); + + assertThat(edsTimeoutTask.isCancelled()).isTrue(); + inOrder.verify(backoffPolicy2).nextBackoffNanos(); + assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); + + fakeClock.forwardNanos(200L); + inOrder.verify(mockedDiscoveryService) + .streamAggregatedResources(responseObserverCaptor.capture()); + + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + } + + /** + * Tests sending a streaming LRS RPC for each cluster to report loads for. + */ + @Test + public void reportLoadStatsToServer() { + String clusterName = "cluster-foo.googleapis.com"; + LoadStatsStore loadStatsStore = new LoadStatsStoreImpl(clusterName, null); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(null); + xdsClient.reportClientStats(clusterName, null, loadStatsStore); + LoadReportCall lrsCall = loadReportCalls.poll(); + verify(lrsCall.requestObserver).onNext(requestCaptor.capture()); + assertThat(requestCaptor.getValue().getClusterStatsCount()) + .isEqualTo(0); // initial request + + lrsCall.responseObserver.onNext( + LoadStatsResponse.newBuilder() + .addClusters(clusterName) + .setLoadReportingInterval(Durations.fromNanos(1000L)) + .build()); + fakeClock.forwardNanos(1000L); + verify(lrsCall.requestObserver, times(2)).onNext(requestCaptor.capture()); + ClusterStats report = Iterables.getOnlyElement(requestCaptor.getValue().getClusterStatsList()); + assertThat(report.getClusterName()).isEqualTo(clusterName); + + xdsClient.cancelClientStatsReport(clusterName, null); + fakeClock.forwardNanos(1000L); + verify(lrsCall.requestObserver, times(3)).onNext(requestCaptor.capture()); + assertThat(requestCaptor.getValue().getClusterStatsCount()) + .isEqualTo(0); // no more stats reported + + // See more test on LoadReportClientTest.java + } + + // Simulates the use case of watching clusters/endpoints based on service config resolved by + // LDS/RDS. + private void waitUntilConfigResolved(StreamObserver responseObserver) { + // Client sent an LDS request for resource TARGET_AUTHORITY (Omitted). + + // Management server responses with a listener telling client to do RDS. + Rds rdsConfig = + Rds.newBuilder() + .setConfigSource( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) + .setRouteConfigName("route-foo.googleapis.com") + .build(); + + List listeners = ImmutableList.of( + Any.pack(buildListener(TARGET_AUTHORITY, /* matching resource */ + Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) + ); + DiscoveryResponse ldsResponse = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + responseObserver.onNext(ldsResponse); + + // Client sent an LDS ACK request and an RDS request for resource + // "route-foo.googleapis.com" (Omitted). + + // Management server sends an RDS response. + List routeConfigs = ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", // target route configuration + ImmutableList.of( + buildVirtualHost( + ImmutableList.of(TARGET_AUTHORITY), // matching virtual host + "cluster.googleapis.com"))))); + DiscoveryResponse rdsResponse = + buildDiscoveryResponseV2("0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); + responseObserver.onNext(rdsResponse); + } + + @Test + public void matchHostName_exactlyMatch() { + String pattern = "foo.googleapis.com"; + assertThat(XdsClientImpl.matchHostName("bar.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("fo.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("oo.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("foo.googleapis", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isTrue(); + } + + @Test + public void matchHostName_prefixWildcard() { + String pattern = "*.foo.googleapis.com"; + assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("bar-baz.foo.googleapis", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("bar.foo.googleapis.com", pattern)).isTrue(); + pattern = "*-bar.foo.googleapis.com"; + assertThat(XdsClientImpl.matchHostName("bar.foo.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("baz-bar.foo.googleapis", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("-bar.foo.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("baz-bar.foo.googleapis.com", pattern)) + .isTrue(); + } + + @Test + public void matchHostName_postfixWildCard() { + String pattern = "foo.*"; + assertThat(XdsClientImpl.matchHostName("bar.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("bar.foo.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isTrue(); + assertThat(XdsClientImpl.matchHostName("foo.com", pattern)).isTrue(); + pattern = "foo-*"; + assertThat(XdsClientImpl.matchHostName("bar-.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("foo-", pattern)).isFalse(); + assertThat(XdsClientImpl.matchHostName("foo-bar.com", pattern)).isTrue(); + assertThat(XdsClientImpl.matchHostName("foo-.com", pattern)).isTrue(); + assertThat(XdsClientImpl.matchHostName("foo-bar", pattern)).isTrue(); + } + + @Test + public void findVirtualHostForHostName_exactMatchFirst() { + String hostname = "a.googleapis.com"; + VirtualHost vHost1 = + VirtualHost.newBuilder() + .setName("virtualhost01.googleapis.com") // don't care + .addAllDomains(ImmutableList.of("a.googleapis.com", "b.googleapis.com")) + .build(); + VirtualHost vHost2 = + VirtualHost.newBuilder() + .setName("virtualhost02.googleapis.com") // don't care + .addAllDomains(ImmutableList.of("*.googleapis.com")) + .build(); + VirtualHost vHost3 = + VirtualHost.newBuilder() + .setName("virtualhost03.googleapis.com") // don't care + .addAllDomains(ImmutableList.of("*")) + .build(); + RouteConfiguration routeConfig = + RouteConfiguration.newBuilder() + .setName("route-foo.googleapis.com") + .addAllVirtualHosts(ImmutableList.of(vHost1, vHost2, vHost3)) + .build(); + assertThat(XdsClientImpl.findVirtualHostForHostName(routeConfig, hostname)).isEqualTo(vHost1); + } + + @Test + public void findVirtualHostForHostName_preferSuffixDomainOverPrefixDomain() { + String hostname = "a.googleapis.com"; + VirtualHost vHost1 = + VirtualHost.newBuilder() + .setName("virtualhost01.googleapis.com") // don't care + .addAllDomains(ImmutableList.of("*.googleapis.com", "b.googleapis.com")) + .build(); + VirtualHost vHost2 = + VirtualHost.newBuilder() + .setName("virtualhost02.googleapis.com") // don't care + .addAllDomains(ImmutableList.of("a.googleapis.*")) + .build(); + VirtualHost vHost3 = + VirtualHost.newBuilder() + .setName("virtualhost03.googleapis.com") // don't care + .addAllDomains(ImmutableList.of("*")) + .build(); + RouteConfiguration routeConfig = + RouteConfiguration.newBuilder() + .setName("route-foo.googleapis.com") + .addAllVirtualHosts(ImmutableList.of(vHost1, vHost2, vHost3)) + .build(); + assertThat(XdsClientImpl.findVirtualHostForHostName(routeConfig, hostname)).isEqualTo(vHost1); + } + + @Test + public void findVirtualHostForHostName_asteriskMatchAnyDomain() { + String hostname = "a.googleapis.com"; + VirtualHost vHost1 = + VirtualHost.newBuilder() + .setName("virtualhost01.googleapis.com") // don't care + .addAllDomains(ImmutableList.of("*")) + .build(); + VirtualHost vHost2 = + VirtualHost.newBuilder() + .setName("virtualhost02.googleapis.com") // don't care + .addAllDomains(ImmutableList.of("b.googleapis.com")) + .build(); + RouteConfiguration routeConfig = + RouteConfiguration.newBuilder() + .setName("route-foo.googleapis.com") + .addAllVirtualHosts(ImmutableList.of(vHost1, vHost2)) + .build(); + assertThat(XdsClientImpl.findVirtualHostForHostName(routeConfig, hostname)).isEqualTo(vHost1); + } + + @Test + public void populateRoutesInVirtualHost_routeWithCaseInsensitiveMatch() { + VirtualHost virtualHost = + VirtualHost.newBuilder() + .setName("virtualhost00.googleapis.com") // don't care + .addDomains(TARGET_AUTHORITY) + .addRoutes( + Route.newBuilder() + .setRoute(RouteAction.newBuilder().setCluster("cluster.googleapis.com")) + .setMatch( + RouteMatch.newBuilder() + .setPrefix("") + .setCaseSensitive(BoolValue.newBuilder().setValue(false)))) + .build(); + + thrown.expect(XdsClientImpl.InvalidProtoDataException.class); + XdsClientImpl.populateRoutesInVirtualHost(virtualHost); + } + + @Test + public void populateRoutesInVirtualHost_NoUsableRoute() { + VirtualHost virtualHost = + VirtualHost.newBuilder() + .setName("virtualhost00.googleapis.com") // don't care + .addDomains(TARGET_AUTHORITY) + .addRoutes( + // route with unsupported action + Route.newBuilder() + .setRoute(RouteAction.newBuilder().setClusterHeader("cluster header string")) + .setMatch(RouteMatch.newBuilder().setPrefix("/"))) + .addRoutes( + // route with unsupported matcher type + Route.newBuilder() + .setRoute(RouteAction.newBuilder().setCluster("cluster.googleapis.com")) + .setMatch( + RouteMatch.newBuilder() + .setPrefix("/") + .addQueryParameters(QueryParameterMatcher.getDefaultInstance()))) + .build(); + + thrown.expect(XdsClientImpl.InvalidProtoDataException.class); + XdsClientImpl.populateRoutesInVirtualHost(virtualHost); + } + + @Test + public void messagePrinter_printLdsResponse() { + MessagePrinter printer = new MessagePrinter(); + List listeners = ImmutableList.of( + Any.pack(buildListener("foo.googleapis.com:8080", + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig( + buildRouteConfiguration("route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("foo.googleapis.com", "bar.googleapis.com"), + "cluster.googleapis.com")))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); + + String expectedString = "{\n" + + " \"versionInfo\": \"0\",\n" + + " \"resources\": [{\n" + + " \"@type\": \"type.googleapis.com/envoy.api.v2.Listener\",\n" + + " \"name\": \"foo.googleapis.com:8080\",\n" + + " \"address\": {\n" + + " },\n" + + " \"filterChains\": [{\n" + + " }],\n" + + " \"apiListener\": {\n" + + " \"apiListener\": {\n" + + " \"@type\": \"type.googleapis.com/envoy.config.filter.network" + + ".http_connection_manager.v2.HttpConnectionManager\",\n" + + " \"routeConfig\": {\n" + + " \"name\": \"route-foo.googleapis.com\",\n" + + " \"virtualHosts\": [{\n" + + " \"name\": \"virtualhost00.googleapis.com\",\n" + + " \"domains\": [\"foo.googleapis.com\", \"bar.googleapis.com\"],\n" + + " \"routes\": [{\n" + + " \"match\": {\n" + + " \"prefix\": \"\"\n" + + " },\n" + + " \"route\": {\n" + + " \"cluster\": \"cluster.googleapis.com\"\n" + + " }\n" + + " }]\n" + + " }]\n" + + " }\n" + + " }\n" + + " }\n" + + " }],\n" + + " \"typeUrl\": \"type.googleapis.com/envoy.api.v2.Listener\",\n" + + " \"nonce\": \"0000\"\n" + + "}"; + String res = printer.print(response); + assertThat(res).isEqualTo(expectedString); + } + + @Test + public void messagePrinter_printRdsResponse() { + MessagePrinter printer = new MessagePrinter(); + List routeConfigs = + ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("foo.googleapis.com", "bar.googleapis.com"), + "cluster.googleapis.com"))))); + DiscoveryResponse response = + buildDiscoveryResponseV2("213", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0052"); + + String expectedString = "{\n" + + " \"versionInfo\": \"213\",\n" + + " \"resources\": [{\n" + + " \"@type\": \"type.googleapis.com/envoy.api.v2.RouteConfiguration\",\n" + + " \"name\": \"route-foo.googleapis.com\",\n" + + " \"virtualHosts\": [{\n" + + " \"name\": \"virtualhost00.googleapis.com\",\n" + + " \"domains\": [\"foo.googleapis.com\", \"bar.googleapis.com\"],\n" + + " \"routes\": [{\n" + + " \"match\": {\n" + + " \"prefix\": \"\"\n" + + " },\n" + + " \"route\": {\n" + + " \"cluster\": \"cluster.googleapis.com\"\n" + + " }\n" + + " }]\n" + + " }]\n" + + " }],\n" + + " \"typeUrl\": \"type.googleapis.com/envoy.api.v2.RouteConfiguration\",\n" + + " \"nonce\": \"0052\"\n" + + "}"; + String res = printer.print(response); + assertThat(res).isEqualTo(expectedString); + } + + @Test + public void messagePrinter_printCdsResponse() { + MessagePrinter printer = new MessagePrinter(); + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-bar.googleapis.com", "service-blaze:cluster-bar", true)), + Any.pack(buildCluster("cluster-foo.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponseV2("14", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "8"); + + String expectedString = "{\n" + + " \"versionInfo\": \"14\",\n" + + " \"resources\": [{\n" + + " \"@type\": \"type.googleapis.com/envoy.api.v2.Cluster\",\n" + + " \"name\": \"cluster-bar.googleapis.com\",\n" + + " \"type\": \"EDS\",\n" + + " \"edsClusterConfig\": {\n" + + " \"edsConfig\": {\n" + + " \"ads\": {\n" + + " }\n" + + " },\n" + + " \"serviceName\": \"service-blaze:cluster-bar\"\n" + + " },\n" + + " \"lrsServer\": {\n" + + " \"self\": {\n" + + " }\n" + + " }\n" + + " }, {\n" + + " \"@type\": \"type.googleapis.com/envoy.api.v2.Cluster\",\n" + + " \"name\": \"cluster-foo.googleapis.com\",\n" + + " \"type\": \"EDS\",\n" + + " \"edsClusterConfig\": {\n" + + " \"edsConfig\": {\n" + + " \"ads\": {\n" + + " }\n" + + " }\n" + + " }\n" + + " }],\n" + + " \"typeUrl\": \"type.googleapis.com/envoy.api.v2.Cluster\",\n" + + " \"nonce\": \"8\"\n" + + "}"; + String res = printer.print(response); + assertThat(res).isEqualTo(expectedString); + } + + @Test + public void messagePrinter_printEdsResponse() { + MessagePrinter printer = new MessagePrinter(); + List clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0), + buildLocalityLbEndpoints("region3", "zone3", "subzone3", + ImmutableList.of( + buildLbEndpoint("192.168.142.5", 80, HealthStatus.UNHEALTHY, 5)), + 2, 1)), + ImmutableList.of( + buildDropOverload("lb", 200), + buildDropOverload("throttle", 1000))))); + + DiscoveryResponse response = + buildDiscoveryResponseV2("5", clusterLoadAssignments, + XdsClientImpl.ADS_TYPE_URL_EDS_V2, "004"); + + String expectedString = "{\n" + + " \"versionInfo\": \"5\",\n" + + " \"resources\": [{\n" + + " \"@type\": \"type.googleapis.com/envoy.api.v2.ClusterLoadAssignment\",\n" + + " \"clusterName\": \"cluster-foo.googleapis.com\",\n" + + " \"endpoints\": [{\n" + + " \"locality\": {\n" + + " \"region\": \"region1\",\n" + + " \"zone\": \"zone1\",\n" + + " \"subZone\": \"subzone1\"\n" + + " },\n" + + " \"lbEndpoints\": [{\n" + + " \"endpoint\": {\n" + + " \"address\": {\n" + + " \"socketAddress\": {\n" + + " \"address\": \"192.168.0.1\",\n" + + " \"portValue\": 8080\n" + + " }\n" + + " }\n" + + " },\n" + + " \"healthStatus\": \"HEALTHY\",\n" + + " \"loadBalancingWeight\": 2\n" + + " }],\n" + + " \"loadBalancingWeight\": 1\n" + + " }, {\n" + + " \"locality\": {\n" + + " \"region\": \"region3\",\n" + + " \"zone\": \"zone3\",\n" + + " \"subZone\": \"subzone3\"\n" + + " },\n" + + " \"lbEndpoints\": [{\n" + + " \"endpoint\": {\n" + + " \"address\": {\n" + + " \"socketAddress\": {\n" + + " \"address\": \"192.168.142.5\",\n" + + " \"portValue\": 80\n" + + " }\n" + + " }\n" + + " },\n" + + " \"healthStatus\": \"UNHEALTHY\",\n" + + " \"loadBalancingWeight\": 5\n" + + " }],\n" + + " \"loadBalancingWeight\": 2,\n" + + " \"priority\": 1\n" + + " }],\n" + + " \"policy\": {\n" + + " \"dropOverloads\": [{\n" + + " \"category\": \"lb\",\n" + + " \"dropPercentage\": {\n" + + " \"numerator\": 200,\n" + + " \"denominator\": \"MILLION\"\n" + + " }\n" + + " }, {\n" + + " \"category\": \"throttle\",\n" + + " \"dropPercentage\": {\n" + + " \"numerator\": 1000,\n" + + " \"denominator\": \"MILLION\"\n" + + " }\n" + + " }],\n" + + " \"disableOverprovisioning\": true\n" + + " }\n" + + " }],\n" + + " \"typeUrl\": \"type.googleapis.com/envoy.api.v2.ClusterLoadAssignment\",\n" + + " \"nonce\": \"004\"\n" + + "}"; + String res = printer.print(response); + assertThat(res).isEqualTo(expectedString); + } + + private static void assertConfigUpdateContainsSingleClusterRoute( + ConfigUpdate configUpdate, String expectedClusterName) { + List routes = configUpdate.getRoutes(); + assertThat(routes).hasSize(1); + assertThat(Iterables.getOnlyElement(routes).getRouteAction().getCluster()) + .isEqualTo(expectedClusterName); + } + + /** + * Matcher for DiscoveryRequest without the comparison of error_details field, which is used for + * management server debugging purposes. + * + *

In general, if you are sure error_details field should not be set in a DiscoveryRequest, + * compare with message equality. Otherwise, this matcher is handy for comparing other fields + * only. + */ + private static class DiscoveryRequestMatcher implements ArgumentMatcher { + private final String versionInfo; + private final String typeUrl; + private final Set resourceNames; + private final String responseNonce; + + private DiscoveryRequestMatcher(String versionInfo, String resourceName, String typeUrl, + String responseNonce) { + this(versionInfo, ImmutableList.of(resourceName), typeUrl, responseNonce); + } + + private DiscoveryRequestMatcher( + String versionInfo, List resourceNames, String typeUrl, String responseNonce) { + this.versionInfo = versionInfo; + this.resourceNames = new HashSet<>(resourceNames); + this.typeUrl = typeUrl; + this.responseNonce = responseNonce; + } + + @Override + public boolean matches(DiscoveryRequest argument) { + if (!typeUrl.equals(argument.getTypeUrl())) { + return false; + } + if (!versionInfo.equals(argument.getVersionInfo())) { + return false; + } + if (!responseNonce.equals(argument.getResponseNonce())) { + return false; + } + if (!resourceNames.equals(new HashSet<>(argument.getResourceNamesList()))) { + return false; + } + return argument.getNode().equals(NODE.toEnvoyProtoNodeV2()); + } + } + + private static class LoadReportCall { + private final StreamObserver requestObserver; + @SuppressWarnings("unused") + private final StreamObserver responseObserver; + + LoadReportCall(StreamObserver requestObserver, + StreamObserver responseObserver) { + this.requestObserver = requestObserver; + this.responseObserver = responseObserver; + } + } +} diff --git a/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java index 9918a9073e1..ab976e877e1 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java @@ -25,8 +25,6 @@ import io.envoyproxy.envoy.api.v2.Cluster.LbPolicy; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment.Policy; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; import io.envoyproxy.envoy.api.v2.Listener; import io.envoyproxy.envoy.api.v2.RouteConfiguration; import io.envoyproxy.envoy.api.v2.auth.CommonTlsContext; @@ -48,6 +46,8 @@ import io.envoyproxy.envoy.api.v2.route.RouteMatch; import io.envoyproxy.envoy.api.v2.route.VirtualHost; import io.envoyproxy.envoy.config.listener.v2.ApiListener; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; import io.envoyproxy.envoy.type.FractionalPercent; import io.envoyproxy.envoy.type.FractionalPercent.DenominatorType; import io.grpc.xds.EnvoyProtoData.Node; @@ -69,6 +69,17 @@ static DiscoveryResponse buildDiscoveryResponse(String versionInfo, .build(); } + static io.envoyproxy.envoy.api.v2.DiscoveryResponse buildDiscoveryResponseV2(String versionInfo, + List resources, String typeUrl, String nonce) { + return + io.envoyproxy.envoy.api.v2.DiscoveryResponse.newBuilder() + .setVersionInfo(versionInfo) + .setTypeUrl(typeUrl) + .addAllResources(resources) + .setNonce(nonce) + .build(); + } + static DiscoveryRequest buildDiscoveryRequest(Node node, String versionInfo, String resourceName, String typeUrl, String nonce) { return buildDiscoveryRequest(node, versionInfo, ImmutableList.of(resourceName), typeUrl, nonce); @@ -78,6 +89,24 @@ static DiscoveryRequest buildDiscoveryRequest(Node node, String versionInfo, List resourceNames, String typeUrl, String nonce) { return DiscoveryRequest.newBuilder() + .setVersionInfo(versionInfo) + .setNode(node.toEnvoyProtoNode()) + .setTypeUrl(typeUrl) + .addAllResourceNames(resourceNames) + .setResponseNonce(nonce) + .build(); + } + + static io.envoyproxy.envoy.api.v2.DiscoveryRequest buildDiscoveryRequestV2( + Node node, String versionInfo, String resourceName, String typeUrl, String nonce) { + return buildDiscoveryRequestV2( + node, versionInfo, ImmutableList.of(resourceName), typeUrl, nonce); + } + + static io.envoyproxy.envoy.api.v2.DiscoveryRequest buildDiscoveryRequestV2( + Node node, String versionInfo, List resourceNames, String typeUrl, String nonce) { + return + io.envoyproxy.envoy.api.v2.DiscoveryRequest.newBuilder() .setVersionInfo(versionInfo) .setNode(node.toEnvoyProtoNodeV2()) .setTypeUrl(typeUrl) diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java index 804c50f5106..7007f74ecb8 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java @@ -17,7 +17,7 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponse; +import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponseV2; import static io.grpc.xds.XdsClientTestHelper.buildListener; import static io.grpc.xds.XdsClientTestHelper.buildRouteConfiguration; import static io.grpc.xds.XdsClientTestHelper.buildVirtualHost; @@ -396,7 +396,7 @@ public void resolve_xdsRoutingLoadBalancing() { List listeners = ImmutableList.of(Any.pack(buildListener(AUTHORITY, Any.pack(httpConnectionManager)))); responseObserver.onNext( - buildDiscoveryResponse("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000")); + buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000")); verify(mockListener).onResult(resolutionResultCaptor.capture()); ResolutionResult result = resolutionResultCaptor.getValue(); @@ -478,7 +478,7 @@ public void resolve_weightedTargetLoadBalancing() { buildVirtualHostForRoutes( AUTHORITY, ImmutableList.of(weightedClustersDefaultRoute)))))); responseObserver.onNext( - buildDiscoveryResponse("0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000")); + buildDiscoveryResponseV2("0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000")); verify(mockListener).onResult(resolutionResultCaptor.capture()); ResolutionResult result = resolutionResultCaptor.getValue(); @@ -547,7 +547,8 @@ private static DiscoveryResponse buildLdsResponseForCluster( ImmutableList.of(host), // exact match clusterName)))) .build())))); - return buildDiscoveryResponse(versionInfo, listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, nonce); + return buildDiscoveryResponseV2( + versionInfo, listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, nonce); } /** @@ -569,7 +570,8 @@ private static DiscoveryResponse buildLdsResponseForRdsResource( Any.pack( buildListener( host, Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build())))); - return buildDiscoveryResponse(versionInfo, listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, nonce); + return buildDiscoveryResponseV2( + versionInfo, listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, nonce); } /** @@ -588,7 +590,7 @@ private static DiscoveryResponse buildRdsResponseForCluster( routeConfigName, ImmutableList.of( buildVirtualHost(ImmutableList.of(host), clusterName))))); - return buildDiscoveryResponse( + return buildDiscoveryResponseV2( versionInfo, routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, nonce); } From 4664f2b6d741453407d45dc41609f35652fc4892 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Sat, 1 Aug 2020 01:48:52 +0000 Subject: [PATCH 8/9] xds: resolve conflicts by adding timeout field in parsed RouteAction for v2 tests (#7278) --- xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java index f2ec57c8f2b..178fe46bd42 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java @@ -752,7 +752,8 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new io.grpc.xds.RouteMatch( /* prefix= */ null, /* path= */ "/service1/method1"), - new EnvoyProtoData.RouteAction("cl1.googleapis.com", null))); + new EnvoyProtoData.RouteAction( + TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); assertThat(routes.get(1)).isEqualTo( new EnvoyProtoData.Route( // path match with weighted cluster route @@ -760,6 +761,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { /* prefix= */ null, /* path= */ "/service2/method2"), new EnvoyProtoData.RouteAction( + TimeUnit.SECONDS.toNanos(15L), null, ImmutableList.of( new EnvoyProtoData.ClusterWeight("cl21.googleapis.com", 30), @@ -771,7 +773,8 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new io.grpc.xds.RouteMatch( /* prefix= */ "/service1/", /* path= */ null), - new EnvoyProtoData.RouteAction("cl1.googleapis.com", null))); + new EnvoyProtoData.RouteAction( + TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); assertThat(routes.get(3)).isEqualTo( new EnvoyProtoData.Route( // default match with cluster route @@ -779,7 +782,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { /* prefix= */ "", /* path= */ null), new EnvoyProtoData.RouteAction( - "cluster.googleapis.com", null))); + TimeUnit.SECONDS.toNanos(15L), "cluster.googleapis.com", null))); } /** From 08f3a0a14b62e6b59cc0accabb5dd16045b20f6c Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Tue, 11 Aug 2020 17:58:03 +0000 Subject: [PATCH 9/9] xds: manage load stats for all clusters in XdsClient (#7299) Move the creation of LoadStatsStore (aka, the stats object) into XdsClient. The XdsClient is responsible for managing the lifetime of stats objects. Creations of LoadStatsStores are reference counted so that multiple EDS policies can retrieve the same stats object for load recording. Counters for recording loads per locality also need to be reference counted, as each EDS policy for the same cluster will receive endpoints for the same group of localities, they will use the same load counters for recording each locality's loads. --- .../java/io/grpc/xds/ClientLoadCounter.java | 45 ++-- .../java/io/grpc/xds/EdsLoadBalancer.java | 33 +-- .../java/io/grpc/xds/LoadReportClient.java | 134 ++-------- .../java/io/grpc/xds/LoadStatsManager.java | 164 ++++++++++++ .../main/java/io/grpc/xds/LoadStatsStore.java | 107 -------- .../java/io/grpc/xds/LoadStatsStoreImpl.java | 61 +++-- .../main/java/io/grpc/xds/LocalityStore.java | 8 +- .../java/io/grpc/xds/LrsLoadBalancer.java | 4 +- .../java/io/grpc/xds/ReferenceCounted.java | 60 +++++ .../main/java/io/grpc/xds/XdsAttributes.java | 1 + xds/src/main/java/io/grpc/xds/XdsClient.java | 32 ++- .../main/java/io/grpc/xds/XdsClientImpl.java | 51 ++-- .../java/io/grpc/xds/CdsLoadBalancerTest.java | 51 ---- .../io/grpc/xds/ClientLoadCounterTest.java | 8 +- .../java/io/grpc/xds/EdsLoadBalancerTest.java | 1 + .../io/grpc/xds/LoadReportClientTest.java | 242 +++++++++++------- .../io/grpc/xds/LoadStatsStoreImplTest.java | 104 +++----- .../java/io/grpc/xds/LocalityStoreTest.java | 34 ++- .../java/io/grpc/xds/LrsLoadBalancerTest.java | 17 +- .../java/io/grpc/xds/XdsClientImplTest.java | 18 +- .../java/io/grpc/xds/XdsClientImplTestV2.java | 18 +- 21 files changed, 606 insertions(+), 587 deletions(-) create mode 100644 xds/src/main/java/io/grpc/xds/LoadStatsManager.java delete mode 100644 xds/src/main/java/io/grpc/xds/LoadStatsStore.java create mode 100644 xds/src/main/java/io/grpc/xds/ReferenceCounted.java diff --git a/xds/src/main/java/io/grpc/xds/ClientLoadCounter.java b/xds/src/main/java/io/grpc/xds/ClientLoadCounter.java index 8e8754d2542..f86002b683f 100644 --- a/xds/src/main/java/io/grpc/xds/ClientLoadCounter.java +++ b/xds/src/main/java/io/grpc/xds/ClientLoadCounter.java @@ -54,27 +54,10 @@ final class ClientLoadCounter { private final AtomicLong callsIssued = new AtomicLong(); private final MetricRecorder[] metricRecorders = new MetricRecorder[THREAD_BALANCING_FACTOR]; - // True if this counter continues to record stats after next snapshot. Otherwise, it will be - // discarded. - private boolean active; - ClientLoadCounter() { for (int i = 0; i < THREAD_BALANCING_FACTOR; i++) { metricRecorders[i] = new MetricRecorder(); } - active = true; - } - - /** - * Must only be used for testing. - */ - @VisibleForTesting - ClientLoadCounter(long callsSucceeded, long callsInProgress, long callsFailed, long callsIssued) { - this(); - this.callsSucceeded.set(callsSucceeded); - this.callsInProgress.set(callsInProgress); - this.callsFailed.set(callsFailed); - this.callsIssued.set(callsIssued); } void recordCallStarted() { @@ -98,12 +81,8 @@ void recordMetric(String name, double value) { } /** - * Generates a snapshot for load stats recorded in this counter. Successive snapshots represent - * load stats recorded for the interval since the previous snapshot. So taking a snapshot clears - * the counter state except for ongoing RPC recordings. - * - *

This method is not thread-safe and must be called from {@link - * io.grpc.LoadBalancer.Helper#getSynchronizationContext()}. + * Generates a snapshot for load stats recorded in this counter for the interval between calls + * of this method. */ ClientLoadSnapshot snapshot() { Map aggregatedValues = new HashMap<>(); @@ -127,12 +106,24 @@ ClientLoadSnapshot snapshot() { aggregatedValues); } - void setActive(boolean value) { - active = value; + @VisibleForTesting + void setCallsIssued(long callsIssued) { + this.callsIssued.set(callsIssued); + } + + @VisibleForTesting + void setCallsInProgress(long callsInProgress) { + this.callsInProgress.set(callsInProgress); } - boolean isActive() { - return active; + @VisibleForTesting + void setCallsSucceeded(long callsSucceeded) { + this.callsSucceeded.set(callsSucceeded); + } + + @VisibleForTesting + void setCallsFailed(long callsFailed) { + this.callsFailed.set(callsFailed); } /** diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java index 5c5c1889fca..b009ea7d530 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java @@ -38,6 +38,7 @@ import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; import io.grpc.xds.EnvoyProtoData.Node; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.LocalityStore.LocalityStoreFactory; import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.EndpointWatcher; @@ -208,11 +209,9 @@ public void shutdown() { */ private final class ClusterEndpointsBalancerFactory extends LoadBalancer.Factory { @Nullable final String clusterServiceName; - final LoadStatsStore loadStatsStore; ClusterEndpointsBalancerFactory(@Nullable String clusterServiceName) { this.clusterServiceName = clusterServiceName; - loadStatsStore = new LoadStatsStoreImpl(clusterName, clusterServiceName); } @Override @@ -248,6 +247,7 @@ final class ClusterEndpointsBalancer extends LoadBalancer { ClusterEndpointsBalancer(Helper helper) { this.helper = helper; resourceName = clusterServiceName != null ? clusterServiceName : clusterName; + LoadStatsStore loadStatsStore = xdsClient.addClientStats(clusterName, clusterServiceName); localityStore = localityStoreFactory.newLocalityStore(logId, helper, lbRegistry, loadStatsStore); endpointWatcher = new EndpointWatcherImpl(); @@ -267,22 +267,12 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { throw new AssertionError("Can only report load to the same management server"); } if (!isReportingLoad) { - logger.log( - XdsLogLevel.INFO, - "Start reporting loads for cluster: {0}, cluster_service: {1}", - clusterName, - clusterServiceName); - xdsClient.reportClientStats(clusterName, clusterServiceName, loadStatsStore); + xdsClient.reportClientStats(); isReportingLoad = true; } } else { if (isReportingLoad) { - logger.log( - XdsLogLevel.INFO, - "Stop reporting loads for cluster: {0}, cluster_service: {1}", - clusterName, - clusterServiceName); - xdsClient.cancelClientStatsReport(clusterName, clusterServiceName); + xdsClient.cancelClientStatsReport(); isReportingLoad = false; } } @@ -304,15 +294,11 @@ public boolean canHandleEmptyAddressListFromNameResolution() { @Override public void shutdown() { if (isReportingLoad) { - logger.log( - XdsLogLevel.INFO, - "Stop reporting loads for cluster: {0}, cluster_service: {1}", - clusterName, - clusterServiceName); - xdsClient.cancelClientStatsReport(clusterName, clusterServiceName); + xdsClient.cancelClientStatsReport(); isReportingLoad = false; } localityStore.reset(); + xdsClient.removeClientStats(clusterName, clusterServiceName); xdsClient.cancelEndpointDataWatch(resourceName, endpointWatcher); logger.log( XdsLogLevel.INFO, @@ -365,12 +351,7 @@ public void onEndpointChanged(EndpointUpdate endpointUpdate) { public void onResourceDoesNotExist(String resourceName) { logger.log(XdsLogLevel.INFO, "Resource {0} is unavailable", resourceName); if (isReportingLoad) { - logger.log( - XdsLogLevel.INFO, - "Stop reporting loads for cluster: {0}, cluster_service: {1}", - clusterName, - clusterServiceName); - xdsClient.cancelClientStatsReport(clusterName, clusterServiceName); + xdsClient.cancelClientStatsReport(); isReportingLoad = false; } localityStore.reset(); diff --git a/xds/src/main/java/io/grpc/xds/LoadReportClient.java b/xds/src/main/java/io/grpc/xds/LoadReportClient.java index 1314583cfa7..84d2b340cbd 100644 --- a/xds/src/main/java/io/grpc/xds/LoadReportClient.java +++ b/xds/src/main/java/io/grpc/xds/LoadReportClient.java @@ -27,7 +27,6 @@ import com.google.protobuf.Value; import com.google.protobuf.util.Durations; import io.envoyproxy.envoy.api.v2.core.Node; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; import io.envoyproxy.envoy.service.load_stats.v2.LoadReportingServiceGrpc; import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest; import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsResponse; @@ -39,10 +38,7 @@ import io.grpc.internal.BackoffPolicy; import io.grpc.stub.StreamObserver; import io.grpc.xds.XdsLogger.XdsLogLevel; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -57,6 +53,7 @@ final class LoadReportClient { @VisibleForTesting static final String TARGET_NAME_METADATA_KEY = "PROXYLESS_CLIENT_HOSTNAME"; + private final InternalLogId logId; private final XdsLogger logger; private final ManagedChannel channel; private final Node node; @@ -65,9 +62,8 @@ final class LoadReportClient { private final Supplier stopwatchSupplier; private final Stopwatch retryStopwatch; private final BackoffPolicy.Provider backoffPolicyProvider; + private final LoadStatsManager loadStatsManager; - // Sources of load stats data for each cluster:cluster_service. - private final Map> loadStatsEntities = new HashMap<>(); private boolean started; @Nullable @@ -76,18 +72,17 @@ final class LoadReportClient { private ScheduledHandle lrsRpcRetryTimer; @Nullable private LrsStream lrsStream; - @Nullable - private LoadReportCallback callback; LoadReportClient( - InternalLogId logId, String targetName, + LoadStatsManager loadStatsManager, ManagedChannel channel, Node node, SynchronizationContext syncContext, ScheduledExecutorService scheduledExecutorService, BackoffPolicy.Provider backoffPolicyProvider, Supplier stopwatchSupplier) { + this.loadStatsManager = checkNotNull(loadStatsManager, "loadStatsManager"); this.channel = checkNotNull(channel, "channel"); this.syncContext = checkNotNull(syncContext, "syncContext"); this.timerService = checkNotNull(scheduledExecutorService, "timeService"); @@ -104,8 +99,9 @@ final class LoadReportClient { Value.newBuilder().setStringValue(targetName).build()) .build(); this.node = node.toBuilder().setMetadata(metadata).build(); - String logPrefix = checkNotNull(logId, "logId").toString().concat("-lrs-client"); - logger = XdsLogger.withPrefix(logPrefix); + logId = InternalLogId.allocate("lrs-client", targetName); + logger = XdsLogger.withLogId(logId); + logger.log(XdsLogLevel.INFO, "Created"); } /** @@ -113,12 +109,12 @@ final class LoadReportClient { * stats periodically. Calling this method on an already started {@link LoadReportClient} is * no-op. */ - void startLoadReporting(LoadReportCallback callback) { + void startLoadReporting() { if (started) { return; } - this.callback = callback; started = true; + logger.log(XdsLogLevel.INFO, "Starting load reporting RPC"); startLrsRpc(); } @@ -130,6 +126,7 @@ void stopLoadReporting() { if (!started) { return; } + logger.log(XdsLogLevel.INFO, "Stopping load reporting RPC"); if (lrsRpcRetryTimer != null) { lrsRpcRetryTimer.cancel(); } @@ -140,49 +137,6 @@ void stopLoadReporting() { // Do not shutdown channel as it is not owned by LrsClient. } - /** - * Provides this LoadReportClient source of load stats data for the given - * cluster:cluster_service. If requested, data from the given loadStatsStore is - * periodically queried and sent to traffic director by this LoadReportClient. - */ - void addLoadStatsStore( - String clusterName, @Nullable String clusterServiceName, LoadStatsStore loadStatsStore) { - checkState( - !loadStatsEntities.containsKey(clusterName) - || !loadStatsEntities.get(clusterName).containsKey(clusterServiceName), - "load stats for cluster: %s, cluster service: %s already exists", - clusterName, clusterServiceName); - logger.log( - XdsLogLevel.INFO, - "Add load stats for cluster: {0}, cluster_service: {1}", clusterName, clusterServiceName); - if (!loadStatsEntities.containsKey(clusterName)) { - loadStatsEntities.put(clusterName, new HashMap()); - } - Map clusterLoadStatsEntities = loadStatsEntities.get(clusterName); - clusterLoadStatsEntities.put(clusterServiceName, new LoadStatsEntity(loadStatsStore)); - } - - /** - * Stops providing load stats data for the given cluster:cluster_service. - */ - void removeLoadStatsStore(String clusterName, @Nullable String clusterServiceName) { - checkState( - loadStatsEntities.containsKey(clusterName) - && loadStatsEntities.get(clusterName).containsKey(clusterServiceName), - "load stats for cluster: %s, cluster service: %s does not exist", - clusterName, clusterServiceName); - logger.log( - XdsLogLevel.INFO, - "Remove load stats for cluster: {0}, cluster_service: {1}", - clusterName, - clusterServiceName); - Map clusterLoadStatsEntities = loadStatsEntities.get(clusterName); - clusterLoadStatsEntities.remove(clusterServiceName); - if (clusterLoadStatsEntities.isEmpty()) { - loadStatsEntities.remove(clusterName); - } - } - @VisibleForTesting static class LoadReportingTask implements Runnable { private final LrsStream stream; @@ -217,12 +171,13 @@ private void startLrsRpc() { private class LrsStream implements StreamObserver { - final Set clusterNames = new HashSet<>(); final LoadReportingServiceGrpc.LoadReportingServiceStub stub; StreamObserver lrsRequestWriter; boolean initialResponseReceived; boolean closed; long loadReportIntervalNano = -1; + boolean reportAllClusters; + List clusterNames; // clusters to report loads for, if not report all. ScheduledHandle loadReportTimer; LrsStream(LoadReportingServiceGrpc.LoadReportingServiceStub stub, Stopwatch stopwatch) { @@ -272,12 +227,11 @@ public void run() { private void sendLoadReport() { LoadStatsRequest.Builder requestBuilder = LoadStatsRequest.newBuilder().setNode(node); - for (String name : clusterNames) { - if (loadStatsEntities.containsKey(name)) { - Map clusterLoadStatsEntities = loadStatsEntities.get(name); - for (LoadStatsEntity entity : clusterLoadStatsEntities.values()) { - requestBuilder.addClusterStats(entity.getLoadStats()); - } + if (reportAllClusters) { + requestBuilder.addAllClusterStats(loadStatsManager.getAllLoadReports()); + } else { + for (String name : clusterNames) { + requestBuilder.addAllClusterStats(loadStatsManager.getClusterLoadReports(name)); } } LoadStatsRequest request = requestBuilder.build(); @@ -309,21 +263,17 @@ private void handleResponse(LoadStatsResponse response) { } else { logger.log(XdsLogLevel.DEBUG, "Received LRS response:\n{0}", response); } - clusterNames.clear(); - if (response.getSendAllClusters()) { - clusterNames.addAll(loadStatsEntities.keySet()); - logger.log(XdsLogLevel.INFO, "Update to report loads for all clusters"); + reportAllClusters = response.getSendAllClusters(); + if (reportAllClusters) { + logger.log(XdsLogLevel.INFO, "Report loads for all clusters"); } else { - logger.log( - XdsLogLevel.INFO, - "Update load reporting clusters to {0}", response.getClustersList()); - clusterNames.addAll(response.getClustersList()); + logger.log(XdsLogLevel.INFO, "Report loads for clusters: ", response.getClustersList()); + clusterNames = response.getClustersList(); } long interval = Durations.toNanos(response.getLoadReportingInterval()); logger.log(XdsLogLevel.INFO, "Update load reporting interval to {0} ns", interval); loadReportIntervalNano = interval; scheduleNextLoadReport(); - callback.onReportResponse(loadReportIntervalNano); } private void handleStreamClosed(Status status) { @@ -385,42 +335,4 @@ private void cleanUp() { } } } - - private final class LoadStatsEntity { - private final LoadStatsStore loadStatsStore; - private final Stopwatch stopwatch; - - private LoadStatsEntity(LoadStatsStore loadStatsStore) { - this.loadStatsStore = loadStatsStore; - this.stopwatch = stopwatchSupplier.get().reset().start(); - } - - private ClusterStats getLoadStats() { - ClusterStats stats = - loadStatsStore.generateLoadReport() - .toBuilder() - .setLoadReportInterval( - Durations.fromNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS))) - .build(); - stopwatch.reset().start(); - return stats; - } - } - - /** - * Callbacks for passing information received from client load reporting responses to xDS load - * balancer, such as the load reporting interval requested by the traffic director. - * - *

Implementations are not required to be thread-safe as callbacks will be invoked in xDS load - * balancer's {@link io.grpc.SynchronizationContext}. - */ - interface LoadReportCallback { - - /** - * The load reporting interval has been received. - * - * @param reportIntervalNano load reporting interval requested by remote traffic director. - */ - void onReportResponse(long reportIntervalNano); - } } diff --git a/xds/src/main/java/io/grpc/xds/LoadStatsManager.java b/xds/src/main/java/io/grpc/xds/LoadStatsManager.java new file mode 100644 index 00000000000..5372d37b0ea --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/LoadStatsManager.java @@ -0,0 +1,164 @@ +/* + * Copyright 2019 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.annotations.VisibleForTesting; +import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; +import io.grpc.xds.EnvoyProtoData.Locality; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Manages all stats for client side load. + */ +final class LoadStatsManager { + private final LoadStatsStoreFactory loadStatsStoreFactory; + private final Map>> loadStatsStores + = new HashMap<>(); + + LoadStatsManager() { + this(LoadStatsStoreImpl.getDefaultFactory()); + } + + @VisibleForTesting + LoadStatsManager(LoadStatsStoreFactory factory) { + this.loadStatsStoreFactory = factory; + } + + /** + * Adds and retrieves the stats object for tracking loads for the given cluster:cluster_service. + * The returned {@link LoadStatsStore} is reference-counted, caller should use + * {@link #removeLoadStats} to release the reference when it is no longer used. + */ + LoadStatsStore addLoadStats(String cluster, @Nullable String clusterService) { + if (!loadStatsStores.containsKey(cluster)) { + loadStatsStores.put(cluster, new HashMap>()); + } + Map> clusterLoadStatsStores + = loadStatsStores.get(cluster); + if (!clusterLoadStatsStores.containsKey(clusterService)) { + clusterLoadStatsStores.put( + clusterService, + ReferenceCounted.wrap(loadStatsStoreFactory.newLoadStatsStore(cluster, clusterService))); + } + ReferenceCounted ref = clusterLoadStatsStores.get(clusterService); + ref.retain(); + return ref.get(); + } + + /** + * Discards stats object used for tracking loads for the given cluster:cluster_service. + */ + void removeLoadStats(String cluster, @Nullable String clusterService) { + checkState( + loadStatsStores.containsKey(cluster) + && loadStatsStores.get(cluster).containsKey(clusterService), + "stats for cluster %s, cluster service %s not exits"); + Map> clusterLoadStatsStores = + loadStatsStores.get(cluster); + ReferenceCounted ref = clusterLoadStatsStores.get(clusterService); + ref.release(); + if (ref.getReferenceCount() == 0) { + clusterLoadStatsStores.remove(clusterService); + } + if (clusterLoadStatsStores.isEmpty()) { + loadStatsStores.remove(cluster); + } + } + + /** + * Generates reports summarizing the stats recorded for loads sent to the given cluster for + * the interval between calls of this method or {@link #getAllLoadReports}. A cluster may send + * loads to more than one cluster_service, they are included in separate stats reports. + */ + // TODO(chengyuanzhang): do not use proto type directly. + List getClusterLoadReports(String cluster) { + List res = new ArrayList<>(); + Map> clusterLoadStatsStores = + loadStatsStores.get(cluster); + if (clusterLoadStatsStores == null) { + return res; + } + for (ReferenceCounted ref : clusterLoadStatsStores.values()) { + res.add(ref.get().generateLoadReport()); + } + return res; + } + + /** + * Generates reports summarized the stats recorded for loads sent to all clusters for the + * interval between calls of this method or {@link #getClusterLoadReports}. Each report + * includes stats for one cluster:cluster_service. + */ + // TODO(chengyuanzhang): do not use proto type directly. + List getAllLoadReports() { + List res = new ArrayList<>(); + for (Map> clusterLoadStatsStores + : loadStatsStores.values()) { + for (ReferenceCounted ref : clusterLoadStatsStores.values()) { + res.add(ref.get().generateLoadReport()); + } + } + return res; + } + + @VisibleForTesting + interface LoadStatsStoreFactory { + LoadStatsStore newLoadStatsStore(String cluster, String clusterService); + } + + /** + * Interface for client side load stats store. An {@code LoadStatsStore} maintains load stats per + * cluster:cluster_service exposed by traffic director from a gRPC client's perspective, + * including dropped calls. Load stats for endpoints are aggregated in locality granularity + * while the numbers of dropped calls are aggregated in cluster:cluster_service granularity. + */ + interface LoadStatsStore { + + /** + * Generates a report based on recorded load stats (including RPC counts, backend metrics and + * dropped calls) for the interval since the previous call of this method. + */ + // TODO(chengyuanzhang): do not use proto type directly. + ClusterStats generateLoadReport(); + + /** + * Track load stats for endpoints in the provided locality. Only load stats for endpoints + * in tracked localities will be included in generated load reports. + */ + ClientLoadCounter addLocality(Locality locality); + + /** + * Drop tracking load stats for endpoints in the provided locality. Load stats for endpoints + * in removed localities will no longer be included in future generated load reports after + * their currently recording stats have been fully reported. + */ + void removeLocality(Locality locality); + + /** + * Records a drop decision. + * + *

This method is thread-safe. + */ + void recordDroppedRequest(String category); + } +} diff --git a/xds/src/main/java/io/grpc/xds/LoadStatsStore.java b/xds/src/main/java/io/grpc/xds/LoadStatsStore.java deleted file mode 100644 index cd76be41edf..00000000000 --- a/xds/src/main/java/io/grpc/xds/LoadStatsStore.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; -import io.grpc.xds.EnvoyProtoData.Locality; -import javax.annotation.Nullable; - -/** - * Interface for client side load stats store. An {@code LoadStatsStore} maintains load stats per - * cluster:cluster_service exposed by traffic director from a gRPC client's perspective, - * including dropped calls. Load stats for endpoints (i.e., Google backends) are aggregated in - * locality granularity (i.e., Google cluster) while the numbers of dropped calls are aggregated - * in cluster:cluster_service granularity. - * - *

An {@code LoadStatsStore} only tracks loads for localities exposed by remote traffic - * director. A proper usage should be - * - *

    - *
  1. Let {@link LoadStatsStore} track the locality newly exposed by traffic director by - * calling {@link #addLocality(Locality)}. - *
  2. Use the locality counter returned by {@link #getLocalityCounter(Locality)} to record - * load stats for the corresponding locality. - *
  3. Tell {@link LoadStatsStore} to stop tracking the locality no longer exposed by traffic - * director by calling {@link #removeLocality(Locality)}. - *
- * - *

No locality information is needed for recording dropped calls since they are aggregated in - * cluster granularity. - */ -interface LoadStatsStore { - - /** - * Generates a {@link ClusterStats} proto message as the load report based on recorded load stats - * (including RPC * counts, backend metrics and dropped calls) for the interval since the previous - * call of this method. - * - *

Loads for localities no longer under tracking will not be included in generated load reports - * once all of theirs loads are completed and reported. - * - *

The fields {@code cluster_name} and {@code load_report_interval} in the returned {@link - * ClusterStats} needs to be set before it is ready to be sent to the traffic director for load - * reporting. - * - *

This method is not thread-safe and should be called from the same synchronized context - * used by {@link LoadReportClient}. - */ - ClusterStats generateLoadReport(); - - /** - * Starts tracking load stats for endpoints in the provided locality. Only load stats for - * endpoints in added localities will be recorded and included in generated load reports. - * - *

This method needs to be called at locality updates only for newly assigned localities in - * endpoint discovery responses before recording loads for those localities. - * - *

This method is not thread-safe and should be called from the same synchronized context - * used by {@link LoadReportClient}. - */ - void addLocality(Locality locality); - - /** - * Stops tracking load stats for endpoints in the provided locality. gRPC clients are expected not - * to send loads to localities no longer exposed by traffic director. Load stats for endpoints in - * removed localities will no longer be included in future generated load reports after their - * recorded and ongoing loads have been reported. - * - *

This method needs to be called at locality updates only for newly removed localities. - * Forgetting calling this method for localities no longer under track will result in memory - * waste and keep including zero-load upstream locality stats in generated load reports. - * - *

This method is not thread-safe and should be called from the same synchronized context - * used by {@link LoadReportClient}. - */ - void removeLocality(Locality locality); - - /** - * Returns the locality counter that does locality level stats aggregation for the provided - * locality. If the provided locality is not tracked, {@code null} will be returned. - * - *

This method is thread-safe. - */ - @Nullable - ClientLoadCounter getLocalityCounter(Locality locality); - - /** - * Records a drop decision made by a {@link io.grpc.LoadBalancer.SubchannelPicker} instance - * with the provided category. Drops are aggregated in cluster granularity. - * - *

This method is thread-safe. - */ - void recordDroppedRequest(String category); -} diff --git a/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java b/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java index 42ba84e8508..ad597d5484c 100644 --- a/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java +++ b/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java @@ -17,19 +17,24 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; +import com.google.protobuf.util.Durations; import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.DroppedRequests; import io.envoyproxy.envoy.api.v2.endpoint.EndpointLoadMetricStats; import io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats; +import io.grpc.internal.GrpcUtil; import io.grpc.xds.ClientLoadCounter.ClientLoadSnapshot; import io.grpc.xds.ClientLoadCounter.MetricValue; import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; +import io.grpc.xds.LoadStatsManager.LoadStatsStoreFactory; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; @@ -45,12 +50,14 @@ final class LoadStatsStoreImpl implements LoadStatsStore { @Nullable @SuppressWarnings("unused") private final String clusterServiceName; - private final ConcurrentMap localityLoadCounters; + private final ConcurrentMap> localityLoadCounters + = new ConcurrentHashMap<>(); // Cluster level dropped request counts for each category decision made by xDS load balancer. private final ConcurrentMap dropCounters; + private final Stopwatch stopwatch; LoadStatsStoreImpl(String clusterName, @Nullable String clusterServiceName) { - this(clusterName, clusterServiceName, new ConcurrentHashMap(), + this(clusterName, clusterServiceName, GrpcUtil.STOPWATCH_SUPPLIER.get(), new ConcurrentHashMap()); } @@ -58,12 +65,13 @@ final class LoadStatsStoreImpl implements LoadStatsStore { LoadStatsStoreImpl( String clusterName, @Nullable String clusterServiceName, - ConcurrentMap localityLoadCounters, + Stopwatch stopwatch, ConcurrentMap dropCounters) { this.clusterName = checkNotNull(clusterName, "clusterName"); this.clusterServiceName = clusterServiceName; - this.localityLoadCounters = checkNotNull(localityLoadCounters, "localityLoadCounters"); + this.stopwatch = checkNotNull(stopwatch, "stopwatch"); this.dropCounters = checkNotNull(dropCounters, "dropCounters"); + stopwatch.reset().start(); } @Override @@ -71,8 +79,9 @@ public ClusterStats generateLoadReport() { ClusterStats.Builder statsBuilder = ClusterStats.newBuilder(); statsBuilder.setClusterName(clusterName); // TODO(chengyuangzhang): also set cluster_service_name if provided. - for (Map.Entry entry : localityLoadCounters.entrySet()) { - ClientLoadSnapshot snapshot = entry.getValue().snapshot(); + for (Map.Entry> entry + : localityLoadCounters.entrySet()) { + ClientLoadSnapshot snapshot = entry.getValue().get().snapshot(); UpstreamLocalityStats.Builder localityStatsBuilder = UpstreamLocalityStats.newBuilder().setLocality(entry.getKey().toEnvoyProtoLocalityV2()); localityStatsBuilder @@ -90,7 +99,7 @@ public ClusterStats generateLoadReport() { statsBuilder.addUpstreamLocalityStats(localityStatsBuilder); // Discard counters for localities that are no longer exposed by the remote balancer and // no RPCs ongoing. - if (!entry.getValue().isActive() && snapshot.getCallsInProgress() == 0) { + if (entry.getValue().getReferenceCount() == 0 && snapshot.getCallsInProgress() == 0) { localityLoadCounters.remove(entry.getKey()); } } @@ -103,32 +112,27 @@ public ClusterStats generateLoadReport() { .setDroppedCount(drops)); } statsBuilder.setTotalDroppedRequests(totalDrops); + statsBuilder.setLoadReportInterval( + Durations.fromNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS))); + stopwatch.reset().start(); return statsBuilder.build(); } @Override - public void addLocality(final Locality locality) { - ClientLoadCounter counter = localityLoadCounters.get(locality); - checkState(counter == null || !counter.isActive(), - "An active counter for locality %s already exists", locality); + public ClientLoadCounter addLocality(final Locality locality) { + ReferenceCounted counter = localityLoadCounters.get(locality); if (counter == null) { - localityLoadCounters.put(locality, new ClientLoadCounter()); - } else { - counter.setActive(true); + counter = ReferenceCounted.wrap(new ClientLoadCounter()); + localityLoadCounters.put(locality, counter); } + counter.retain(); + return counter.get(); } @Override public void removeLocality(final Locality locality) { - ClientLoadCounter counter = localityLoadCounters.get(locality); - checkState(counter != null && counter.isActive(), - "No active counter for locality %s exists", locality); - counter.setActive(false); - } - - @Override - public ClientLoadCounter getLocalityCounter(final Locality locality) { - return localityLoadCounters.get(locality); + ReferenceCounted counter = localityLoadCounters.get(locality); + counter.release(); } @Override @@ -142,4 +146,13 @@ public void recordDroppedRequest(String category) { } counter.getAndIncrement(); } + + static LoadStatsStoreFactory getDefaultFactory() { + return new LoadStatsStoreFactory() { + @Override + public LoadStatsStore newLoadStatsStore(String cluster, String clusterService) { + return new LoadStatsStoreImpl(cluster, clusterService); + } + }; + } } diff --git a/xds/src/main/java/io/grpc/xds/LocalityStore.java b/xds/src/main/java/io/grpc/xds/LocalityStore.java index 183b4bbb7f5..4fa507e6a12 100644 --- a/xds/src/main/java/io/grpc/xds/LocalityStore.java +++ b/xds/src/main/java/io/grpc/xds/LocalityStore.java @@ -47,6 +47,7 @@ import io.grpc.xds.EnvoyProtoData.LbEndpoint; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.OrcaOobUtil.OrcaReportingConfig; import io.grpc.xds.OrcaOobUtil.OrcaReportingHelperWrapper; import io.grpc.xds.WeightedRandomPicker.WeightedChildPicker; @@ -311,8 +312,8 @@ private final class LocalityLbInfo { LocalityLbInfo(Locality locality) { this.locality = checkNotNull(locality, "locality"); - loadStatsStore.addLocality(locality); - childHelper = new ChildHelper(); + ClientLoadCounter counter = loadStatsStore.addLocality(locality); + childHelper = new ChildHelper(counter); childBalancer = loadBalancerProvider.newLoadBalancer(childHelper); } @@ -368,8 +369,7 @@ class ChildHelper extends ForwardingLoadBalancerHelper { private SubchannelPicker currentChildPicker = XdsSubchannelPickers.BUFFER_PICKER; private ConnectivityState currentChildState = CONNECTING; - ChildHelper() { - final ClientLoadCounter counter = loadStatsStore.getLocalityCounter(locality); + ChildHelper(final ClientLoadCounter counter) { Helper delegate = new ForwardingLoadBalancerHelper() { @Override protected Helper delegate() { diff --git a/xds/src/main/java/io/grpc/xds/LrsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/LrsLoadBalancer.java index 55ca451271b..edcfcc0cc39 100644 --- a/xds/src/main/java/io/grpc/xds/LrsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/LrsLoadBalancer.java @@ -26,6 +26,7 @@ import io.grpc.util.GracefulSwitchLoadBalancer; import io.grpc.xds.ClientLoadCounter.LoadRecordingSubchannelPicker; import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.LrsLoadBalancerProvider.LrsConfig; import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.Objects; @@ -58,8 +59,7 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { checkAndSetUp(config, store); if (switchingLoadBalancer == null) { - loadStatsStore.addLocality(config.locality); - final ClientLoadCounter counter = loadStatsStore.getLocalityCounter(config.locality); + final ClientLoadCounter counter = loadStatsStore.addLocality(config.locality); LoadBalancer.Helper loadRecordingHelper = new ForwardingLoadBalancerHelper() { @Override protected Helper delegate() { diff --git a/xds/src/main/java/io/grpc/xds/ReferenceCounted.java b/xds/src/main/java/io/grpc/xds/ReferenceCounted.java new file mode 100644 index 00000000000..b62340ffa0a --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/ReferenceCounted.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + +/** + * A reference count wrapper for objects. This class does not take the ownership for the object, + * but only provides usage counting. The real owner of the wrapped object is responsible for + * managing the lifecycle of the object. + * + *

Intended for a container class to keep track of lifecycle for elements it contains. This + * wrapper itself should never be returned to the consumers of the elements to avoid reference + * counts being leaked. + */ +final class ReferenceCounted { + private final T instance; + private int refs; + + private ReferenceCounted(T instance) { + this.instance = instance; + } + + static ReferenceCounted wrap(T instance) { + checkNotNull(instance, "instance"); + return new ReferenceCounted<>(instance); + } + + void retain() { + refs++; + } + + void release() { + checkState(refs > 0, "reference reached 0"); + refs--; + } + + int getReferenceCount() { + return refs; + } + + T get() { + return instance; + } +} diff --git a/xds/src/main/java/io/grpc/xds/XdsAttributes.java b/xds/src/main/java/io/grpc/xds/XdsAttributes.java index 662c9a69a5a..57796658636 100644 --- a/xds/src/main/java/io/grpc/xds/XdsAttributes.java +++ b/xds/src/main/java/io/grpc/xds/XdsAttributes.java @@ -22,6 +22,7 @@ import io.grpc.NameResolver; import io.grpc.internal.ObjectPool; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; /** * Special attributes that are only useful to gRPC in the XDS context. diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index 1c895c878ab..69a12843003 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -37,6 +37,7 @@ import io.grpc.xds.EnvoyProtoData.Route; import io.grpc.xds.EnvoyServerProtoData.Listener; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.XdsLogger.XdsLogLevel; import java.util.ArrayList; import java.util.Collection; @@ -518,21 +519,34 @@ void watchListenerData(int port, ListenerWatcher watcher) { } /** - * Report client load stats to a remote server for the given cluster:cluster_service. - * - *

Note: currently we can only report loads for a single cluster:cluster_service, - * as the design for adding clusters to report loads for while load reporting is - * happening is undefined. + * Starts client side load reporting via LRS. All clusters report load through one LRS stream, + * only the first call of this method effectively starts the LRS stream. + */ + void reportClientStats() { + } + + /** + * Stops client side load reporting via LRS. All clusters report load through one LRS stream, + * only the last call of this method effectively stops the LRS stream. */ - void reportClientStats( - String clusterName, @Nullable String clusterServiceName, LoadStatsStore loadStatsStore) { + void cancelClientStatsReport() { + } + + /** + * Starts recording client load stats for the given cluster:cluster_service. Caller should use + * the returned {@link LoadStatsStore} to record and aggregate stats for load sent to the given + * cluster:cluster_service. Recorded stats may be reported to a load reporting server if enabled. + */ + LoadStatsStore addClientStats(String clusterName, @Nullable String clusterServiceName) { throw new UnsupportedOperationException(); } /** - * Stops reporting client load stats to the remote server for the given cluster:cluster_service. + * Stops recording client load stats for the given cluster:cluster_service. The load reporting + * server will no longer receive stats for the given cluster:cluster_service after this call. */ - void cancelClientStatsReport(String clusterName, @Nullable String clusterServiceName) { + void removeClientStats(String clusterName, @Nullable String clusterServiceName) { + throw new UnsupportedOperationException(); } abstract static class XdsClientFactory { diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index 621eaa0ce47..c386daca3c6 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -61,7 +61,7 @@ import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.StructOrError; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; -import io.grpc.xds.LoadReportClient.LoadReportCallback; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.XdsLogger.XdsLogLevel; import java.util.ArrayList; import java.util.Collection; @@ -163,6 +163,8 @@ final class XdsClientImpl extends XdsClient { // Timers for concluding EDS resources not found. private final Map edsRespTimers = new HashMap<>(); + private final LoadStatsManager loadStatsManager = new LoadStatsManager(); + // Timer for concluding the currently requesting LDS resource not found. @Nullable private ScheduledHandle ldsRespTimer; @@ -179,6 +181,7 @@ final class XdsClientImpl extends XdsClient { private ScheduledHandle rpcRetryTimer; @Nullable private LoadReportClient lrsClient; + private int loadReportCount; // number of clusters enabling load reporting // Following fields are set only after the ConfigWatcher registered. Once set, they should // never change. Only a ConfigWatcher or ListenerWatcher can be registered. @@ -474,41 +477,45 @@ private void updateNodeMetadataForListenerRequest(int port) { } @Override - void reportClientStats( - String clusterName, @Nullable String clusterServiceName, LoadStatsStore loadStatsStore) { + void reportClientStats() { if (lrsClient == null) { + logger.log(XdsLogLevel.INFO, "Turning on load reporting"); lrsClient = new LoadReportClient( - logId, targetName, + loadStatsManager, channel, node.toEnvoyProtoNodeV2(), syncContext, timeService, backoffPolicyProvider, stopwatchSupplier); - lrsClient.startLoadReporting(new LoadReportCallback() { - @Override - public void onReportResponse(long reportIntervalNano) {} - }); } - logger.log( - XdsLogLevel.INFO, - "Report loads for cluster: {0}, cluster_service: {1}", clusterName, clusterServiceName); - lrsClient.addLoadStatsStore(clusterName, clusterServiceName, loadStatsStore); + if (loadReportCount == 0) { + lrsClient.startLoadReporting(); + } + loadReportCount++; } @Override - void cancelClientStatsReport(String clusterName, @Nullable String clusterServiceName) { - checkState(lrsClient != null, "load reporting was never started"); - logger.log( - XdsLogLevel.INFO, - "Stop reporting loads for cluster: {0}, cluster_service: {1}", - clusterName, - clusterServiceName); - lrsClient.removeLoadStatsStore(clusterName, clusterServiceName); - // TODO(chengyuanzhang): can be optimized to stop load reporting if no more loads need - // to be reported. + void cancelClientStatsReport() { + checkState(loadReportCount > 0, "load reporting was never started"); + loadReportCount--; + if (loadReportCount == 0) { + logger.log(XdsLogLevel.INFO, "Turning off load reporting"); + lrsClient.stopLoadReporting(); + lrsClient = null; + } + } + + @Override + LoadStatsStore addClientStats(String clusterName, @Nullable String clusterServiceName) { + return loadStatsManager.addLoadStats(clusterName, clusterServiceName); + } + + @Override + void removeClientStats(String clusterName, @Nullable String clusterServiceName) { + loadStatsManager.removeLoadStats(clusterName, clusterServiceName); } @Override diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java index 34d57be999b..9a9bb69241f 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java @@ -29,7 +29,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -61,8 +60,6 @@ import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.XdsClient.ClusterUpdate; import io.grpc.xds.XdsClient.ClusterWatcher; -import io.grpc.xds.XdsClient.EndpointUpdate; -import io.grpc.xds.XdsClient.EndpointWatcher; import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; import io.grpc.xds.XdsClient.XdsClientFactory; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; @@ -592,52 +589,4 @@ public void clusterWatcher_onErrorCalledBeforeAndAfterOnClusterChanged() { verify(helper, times(1)) .updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); } - - @Test - public void cdsBalancerIntegrateWithEdsBalancer() { - lbRegistry.deregister(fakeEdsLoadBlancerProvider); - lbRegistry.register(new EdsLoadBalancerProvider()); - - ResolvedAddresses resolvedAddresses1 = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); - ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); - ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); - clusterWatcher.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); - - ArgumentCaptor endpointWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchEndpointData( - eq("edsServiceFoo.googleapis.com"), endpointWatcherCaptor.capture()); - EndpointWatcher endpointWatcher = endpointWatcherCaptor.getValue(); - - verify(helper, never()).updateBalancingState( - eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - // Update endpoints with all backends unhealthy, the EDS will update channel state to - // TRANSIENT_FAILURE. - // Not able to test with healthy endpoints because the real EDS balancer is using real - // round-robin balancer to balance endpoints. - endpointWatcher.onEndpointChanged(EndpointUpdate.newBuilder() - .setClusterName("edsServiceFoo.googleapis.com") - .addLocalityLbEndpoints( - new EnvoyProtoData.Locality("region", "zone", "subzone"), - new EnvoyProtoData.LocalityLbEndpoints( - // All unhealthy. - ImmutableList.of(new EnvoyProtoData.LbEndpoint("127.0.0.1", 8080, 1, false)), 1, 0)) - .build()); - verify(helper, atLeastOnce()).updateBalancingState( - eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - - cdsLoadBalancer.shutdown(); - } } diff --git a/xds/src/test/java/io/grpc/xds/ClientLoadCounterTest.java b/xds/src/test/java/io/grpc/xds/ClientLoadCounterTest.java index bedeaf4eaea..db3fbb818d2 100644 --- a/xds/src/test/java/io/grpc/xds/ClientLoadCounterTest.java +++ b/xds/src/test/java/io/grpc/xds/ClientLoadCounterTest.java @@ -77,9 +77,11 @@ public void snapshotContainsDataInCounter() { long numInProgressCalls = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); long numFailedCalls = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); long numIssuedCalls = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); - counter = - new ClientLoadCounter(numSucceededCalls, numInProgressCalls, numFailedCalls, - numIssuedCalls); + counter = new ClientLoadCounter(); + counter.setCallsSucceeded(numSucceededCalls); + counter.setCallsInProgress(numInProgressCalls); + counter.setCallsFailed(numFailedCalls); + counter.setCallsIssued(numIssuedCalls); ClientLoadSnapshot snapshot = counter.snapshot(); assertQueryCounts(snapshot, numSucceededCalls, numInProgressCalls, numFailedCalls, numIssuedCalls); diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java index 0b923b3033e..2edfd673f72 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java @@ -76,6 +76,7 @@ import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; import io.grpc.xds.EnvoyProtoData.Node; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.LocalityStore.LocalityStoreFactory; import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.XdsChannel; diff --git a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java index 4b8645a7e93..be68f08b110 100644 --- a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java +++ b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.MoreExecutors; @@ -44,7 +45,6 @@ import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsResponse; import io.grpc.Context; import io.grpc.Context.CancellationListener; -import io.grpc.InternalLogId; import io.grpc.ManagedChannel; import io.grpc.Status; import io.grpc.SynchronizationContext; @@ -54,7 +54,8 @@ import io.grpc.internal.FakeClock; import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; -import io.grpc.xds.LoadReportClient.LoadReportCallback; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; +import io.grpc.xds.LoadStatsManager.LoadStatsStoreFactory; import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collection; @@ -62,10 +63,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import javax.annotation.Nullable; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -95,6 +96,8 @@ public class LoadReportClientTest { "TRAFFICDIRECTOR_NETWORK_HOSTNAME", Value.newBuilder().setStringValue("default").build())) .build(); + private static final String CLUSTER1 = "cluster-foo.googleapis.com"; + private static final String CLUSTER2 = "cluster-bar.googleapis.com"; private static final FakeClock.TaskFilter LOAD_REPORTING_TASK_FILTER = new FakeClock.TaskFilter() { @Override @@ -121,11 +124,18 @@ public void uncaughtException(Thread t, Throwable e) { throw new AssertionError(e); } }); - private final InternalLogId logId = InternalLogId.allocate("lrs-client-test", null); private final FakeClock fakeClock = new FakeClock(); private final ArrayDeque> lrsRequestObservers = new ArrayDeque<>(); private final AtomicBoolean callEnded = new AtomicBoolean(true); + private final LoadStatsManager loadStatsManager = + new LoadStatsManager(new LoadStatsStoreFactory() { + @Override + public LoadStatsStore newLoadStatsStore(String cluster, String clusterService) { + return new FakeLoadStatsStore( + cluster, clusterService, fakeClock.getStopwatchSupplier().get()); + } + }); @Mock private BackoffPolicy.Provider backoffPolicyProvider; @@ -133,12 +143,6 @@ public void uncaughtException(Thread t, Throwable e) { private BackoffPolicy backoffPolicy1; @Mock private BackoffPolicy backoffPolicy2; - @Mock - private LoadStatsStore loadStatsStore1; - @Mock - private LoadStatsStore loadStatsStore2; - @Mock - private LoadReportCallback callback; @Captor private ArgumentCaptor> lrsResponseObserverCaptor; @@ -183,15 +187,15 @@ public void cancelled(Context context) { .thenReturn(TimeUnit.SECONDS.toNanos(2L), TimeUnit.SECONDS.toNanos(20L)); lrsClient = new LoadReportClient( - logId, TARGET_NAME, + loadStatsManager, channel, NODE, syncContext, fakeClock.getScheduledExecutorService(), backoffPolicyProvider, fakeClock.getStopwatchSupplier()); - lrsClient.startLoadReporting(callback); + lrsClient.startLoadReporting(); } @After @@ -201,52 +205,55 @@ public void tearDown() { } @Test - public void typicalWorkflow() { + public void periodicLoadReporting() { verify(mockLoadReportingService).streamLoadStats(lrsResponseObserverCaptor.capture()); StreamObserver responseObserver = lrsResponseObserverCaptor.getValue(); StreamObserver requestObserver = Iterables.getOnlyElement(lrsRequestObservers); - InOrder inOrder = inOrder(requestObserver, callback); + InOrder inOrder = inOrder(requestObserver); inOrder.verify(requestObserver).onNext(eq(buildInitialRequest())); - String cluster1 = "cluster-foo.googleapis.com"; - ClusterStats rawStats1 = generateClusterLoadStats(cluster1, null); - when(loadStatsStore1.generateLoadReport()).thenReturn(rawStats1); - lrsClient.addLoadStatsStore(cluster1, null, loadStatsStore1); + FakeLoadStatsStore loadStatsStore1 = + (FakeLoadStatsStore) loadStatsManager.addLoadStats(CLUSTER1, null); + loadStatsStore1.refresh(); // Management server asks to report loads for cluster1. - responseObserver.onNext(buildLrsResponse(ImmutableList.of(cluster1), 1000)); - inOrder.verify(callback).onReportResponse(1000); + responseObserver.onNext(buildLrsResponse(ImmutableList.of(CLUSTER1), 1000)); - ClusterStats expectedStats1 = - rawStats1.toBuilder().setLoadReportInterval(Durations.fromNanos(1000)).build(); fakeClock.forwardNanos(999); inOrder.verifyNoMoreInteractions(); fakeClock.forwardNanos(1); + assertThat(loadStatsStore1.reported).hasSize(1); + ClusterStats report1 = loadStatsStore1.reported.poll(); + assertThat(Durations.toNanos(report1.getLoadReportInterval())).isEqualTo(1000); inOrder.verify(requestObserver) - .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats1)))); + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report1)))); + loadStatsStore1.refresh(); fakeClock.forwardNanos(1000); + assertThat(loadStatsStore1.reported).hasSize(1); + report1 = loadStatsStore1.reported.poll(); + assertThat(Durations.toNanos(report1.getLoadReportInterval())).isEqualTo(1000); inOrder.verify(requestObserver) - .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats1)))); + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report1)))); - String cluster2 = "cluster-bar.googleapis.com"; - ClusterStats rawStats2 = generateClusterLoadStats(cluster2, null); - when(loadStatsStore2.generateLoadReport()).thenReturn(rawStats2); - lrsClient.addLoadStatsStore(cluster2, null, loadStatsStore2); + FakeLoadStatsStore loadStatsStore2 = + (FakeLoadStatsStore) loadStatsManager.addLoadStats(CLUSTER2, null); + loadStatsStore2.refresh(); // Management server updates the interval of sending load reports, while still asking for // loads to cluster1 only. - responseObserver.onNext(buildLrsResponse(ImmutableList.of(cluster1), 2000)); - inOrder.verify(callback).onReportResponse(2000); + responseObserver.onNext(buildLrsResponse(ImmutableList.of(CLUSTER1), 2000)); - expectedStats1 = - rawStats1.toBuilder().setLoadReportInterval(Durations.fromNanos(2000)).build(); fakeClock.forwardNanos(1000); inOrder.verifyNoMoreInteractions(); fakeClock.forwardNanos(1000); + assertThat(loadStatsStore1.reported).hasSize(1); + report1 = loadStatsStore1.reported.poll(); + assertThat(Durations.toNanos(report1.getLoadReportInterval())).isEqualTo(2000); + assertThat(loadStatsStore2.reported).isEmpty(); inOrder.verify(requestObserver) - .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats1)))); + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report1)))); // Management server asks to report loads for all clusters. responseObserver.onNext( @@ -254,30 +261,39 @@ public void typicalWorkflow() { .setSendAllClusters(true) .setLoadReportingInterval(Durations.fromNanos(2000)) .build()); - inOrder.verify(callback).onReportResponse(2000); - ClusterStats expectedStats2 = - rawStats2.toBuilder().setLoadReportInterval(Durations.fromNanos(2000 + 2000)).build(); + loadStatsStore1.refresh(); + loadStatsStore2.refresh(); fakeClock.forwardNanos(2000); + assertThat(loadStatsStore1.reported).hasSize(1); + report1 = loadStatsStore1.reported.poll(); + assertThat(loadStatsStore2.reported).hasSize(1); + ClusterStats report2 = loadStatsStore2.reported.poll(); + assertThat(Durations.toNanos(report1.getLoadReportInterval())).isEqualTo(2000); + assertThat(Durations.toNanos(report2.getLoadReportInterval())).isEqualTo(2000 + 2000); inOrder.verify(requestObserver) - .onNext( - argThat( - new LoadStatsRequestMatcher(Arrays.asList(expectedStats1, expectedStats2)))); + .onNext(argThat(new LoadStatsRequestMatcher(Arrays.asList(report1, report2)))); // Load reports for cluster1 is no longer wanted. - responseObserver.onNext(buildLrsResponse(Collections.singletonList(cluster2), 2000)); + responseObserver.onNext(buildLrsResponse(Collections.singletonList(CLUSTER2), 2000)); - expectedStats2 = - rawStats2.toBuilder().setLoadReportInterval(Durations.fromNanos(2000)).build(); + loadStatsStore1.refresh(); + loadStatsStore2.refresh(); fakeClock.forwardNanos(2000); + assertThat(loadStatsStore1.reported).isEmpty(); + assertThat(loadStatsStore2.reported).hasSize(1); + report2 = loadStatsStore2.reported.poll(); + assertThat(Durations.toNanos(report2.getLoadReportInterval())).isEqualTo(2000); inOrder.verify(requestObserver) - .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats2)))); + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report2)))); // Management server asks loads for a cluster that client has no load data. responseObserver .onNext(buildLrsResponse(ImmutableList.of("cluster-unknown.googleapis.com"), 2000)); fakeClock.forwardNanos(2000); + assertThat(loadStatsStore1.reported).isEmpty(); + assertThat(loadStatsStore2.reported).isEmpty(); ArgumentCaptor reportCaptor = ArgumentCaptor.forClass(null); inOrder.verify(requestObserver).onNext(reportCaptor.capture()); assertThat(reportCaptor.getValue().getClusterStatsCount()).isEqualTo(0); @@ -296,9 +312,9 @@ public void lrsStreamClosedAndRetried() { String clusterName = "cluster-foo.googleapis.com"; String clusterServiceName = "service-blade.googleapis.com"; - ClusterStats stats = generateClusterLoadStats(clusterName, clusterServiceName); - when(loadStatsStore1.generateLoadReport()).thenReturn(stats); - lrsClient.addLoadStatsStore(clusterName, null, loadStatsStore1); + FakeLoadStatsStore loadStatsStore = + (FakeLoadStatsStore) loadStatsManager.addLoadStats(clusterName, clusterServiceName); + loadStatsStore.refresh(); // First balancer RPC verify(requestObserver).onNext(eq(buildInitialRequest())); @@ -382,13 +398,11 @@ public void lrsStreamClosedAndRetried() { responseObserver .onNext(buildLrsResponse(ImmutableList.of(clusterName), 10)); fakeClock.forwardNanos(10); - ClusterStats expectedStats = - stats.toBuilder() - .setLoadReportInterval( - Durations.add(Durations.fromSeconds(1 + 10 + 2), Durations.fromNanos(10))) - .build(); + ClusterStats report = Iterables.getOnlyElement(loadStatsStore.reported); + assertThat(Durations.toNanos(report.getLoadReportInterval())) + .isEqualTo(TimeUnit.SECONDS.toNanos(1 + 10 + 2) + 10); verify(requestObserver) - .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(expectedStats)))); + .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report)))); // Wrapping up verify(backoffPolicyProvider, times(2)).get(); @@ -405,9 +419,9 @@ public void raceBetweenLoadReportingAndLbStreamClosure() { String clusterName = "cluster-foo.googleapis.com"; String clusterServiceName = "service-blade.googleapis.com"; - ClusterStats stats = generateClusterLoadStats(clusterName, clusterServiceName); - when(loadStatsStore1.generateLoadReport()).thenReturn(stats); - lrsClient.addLoadStatsStore(clusterName, null, loadStatsStore1); + FakeLoadStatsStore loadStatsStore = + (FakeLoadStatsStore) loadStatsManager.addLoadStats(clusterName, clusterServiceName); + loadStatsStore.refresh(); // First balancer RPC verify(requestObserver).onNext(eq(buildInitialRequest())); @@ -464,46 +478,6 @@ private static LoadStatsRequest buildInitialRequest() { .build(); } - /** - * Generates a raw service load stats report with random data. - */ - private static ClusterStats generateClusterLoadStats( - String clusterName, @Nullable String clusterServiceName) { - long callsInProgress = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); - long callsSucceeded = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); - long callsFailed = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); - long callsIssued = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); - long numLbDrops = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); - long numThrottleDrops = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); - - ClusterStats.Builder clusterStatsBuilder = ClusterStats.newBuilder(); - clusterStatsBuilder.setClusterName(clusterName); - if (clusterServiceName != null) { - clusterStatsBuilder.setClusterServiceName(clusterServiceName); - } - clusterStatsBuilder.addUpstreamLocalityStats( - UpstreamLocalityStats.newBuilder() - .setLocality( - Locality.newBuilder() - .setRegion(clusterName + "-region-foo") - .setZone(clusterName + "-zone-bar") - .setSubZone(clusterName + "-subzone-baz")) - .setTotalRequestsInProgress(callsInProgress) - .setTotalSuccessfulRequests(callsSucceeded) - .setTotalErrorRequests(callsFailed) - .setTotalIssuedRequests(callsIssued)) - .addDroppedRequests( - DroppedRequests.newBuilder() - .setCategory("lb") - .setDroppedCount(numLbDrops)) - .addDroppedRequests( - DroppedRequests.newBuilder() - .setCategory("throttle") - .setDroppedCount(numThrottleDrops)) - .setTotalDroppedRequests(numLbDrops + numThrottleDrops); - return clusterStatsBuilder.build(); - } - /** * For comparing LoadStatsRequest stats data regardless of . */ @@ -534,4 +508,82 @@ public boolean matches(LoadStatsRequest argument) { return true; } } + + private static final class FakeLoadStatsStore implements LoadStatsStore { + private final String cluster; + private final String clusterService; + private final Stopwatch stopwatch; + private final Queue reported = new ArrayDeque<>(); + private ClusterStats stats; + + private FakeLoadStatsStore(String cluster, String clusterService, Stopwatch stopwatch) { + this.cluster = cluster; + this.clusterService = clusterService; + this.stopwatch = stopwatch; + stopwatch.reset().start(); + refresh(); + } + + @Override + public ClusterStats generateLoadReport() { + ClusterStats report = + stats.toBuilder() + .setLoadReportInterval(Durations.fromNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS))) + .build(); + stopwatch.reset().start(); + reported.offer(report); + return report; + } + + @Override + public ClientLoadCounter addLocality(EnvoyProtoData.Locality locality) { + throw new UnsupportedOperationException("should not used"); + } + + @Override + public void removeLocality(EnvoyProtoData.Locality locality) { + throw new UnsupportedOperationException("should not used"); + } + + @Override + public void recordDroppedRequest(String category) { + throw new UnsupportedOperationException("should not used"); + } + + private void refresh() { + long callsInProgress = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); + long callsSucceeded = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); + long callsFailed = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); + long callsIssued = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); + long numLbDrops = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); + long numThrottleDrops = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); + + ClusterStats.Builder clusterStatsBuilder = ClusterStats.newBuilder(); + clusterStatsBuilder.setClusterName(cluster); + if (clusterService != null) { + clusterStatsBuilder.setClusterServiceName(clusterService); + } + clusterStatsBuilder.addUpstreamLocalityStats( + UpstreamLocalityStats.newBuilder() + .setLocality( + Locality.newBuilder() + .setRegion(cluster + "-region-foo") + .setZone(cluster + "-zone-bar") + .setSubZone(cluster + "-subzone-baz")) + .setTotalRequestsInProgress(callsInProgress) + .setTotalSuccessfulRequests(callsSucceeded) + .setTotalErrorRequests(callsFailed) + .setTotalIssuedRequests(callsIssued)) + .addDroppedRequests( + DroppedRequests.newBuilder() + .setCategory("lb") + .setDroppedCount(numLbDrops)) + .addDroppedRequests( + DroppedRequests.newBuilder() + .setCategory("throttle") + .setDroppedCount(numThrottleDrops)) + .setTotalDroppedRequests(numLbDrops + numThrottleDrops); + stats = clusterStatsBuilder.build(); + } + } } diff --git a/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java b/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java index 96b96b67684..4320ff279fd 100644 --- a/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java +++ b/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java @@ -18,13 +18,17 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; +import com.google.protobuf.util.Durations; import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.DroppedRequests; import io.envoyproxy.envoy.api.v2.endpoint.EndpointLoadMetricStats; import io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats; +import io.grpc.internal.FakeClock; import io.grpc.xds.ClientLoadCounter.MetricValue; import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -35,7 +39,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,16 +52,15 @@ public class LoadStatsStoreImplTest { new Locality("test_region1", "test_zone", "test_subzone"); private static final Locality LOCALITY2 = new Locality("test_region2", "test_zone", "test_subzone"); - private ConcurrentMap localityLoadCounters; + private final FakeClock fakeClock = new FakeClock(); private ConcurrentMap dropCounters; private LoadStatsStore loadStatsStore; @Before public void setUp() { - localityLoadCounters = new ConcurrentHashMap<>(); dropCounters = new ConcurrentHashMap<>(); - loadStatsStore = - new LoadStatsStoreImpl(CLUSTER_NAME, null, localityLoadCounters, dropCounters); + Stopwatch stopwatch = fakeClock.getStopwatchSupplier().get(); + loadStatsStore = new LoadStatsStoreImpl(CLUSTER_NAME, null, stopwatch, dropCounters); } private static List buildEndpointLoadMetricStatsList( @@ -103,7 +105,7 @@ private static DroppedRequests buildDroppedRequests(String category, long counts private static ClusterStats buildClusterStats( @Nullable List upstreamLocalityStatsList, - @Nullable List droppedRequestsList) { + @Nullable List droppedRequestsList, long intervalNano) { ClusterStats.Builder clusterStatsBuilder = ClusterStats.newBuilder(); clusterStatsBuilder.setClusterName(CLUSTER_NAME); if (upstreamLocalityStatsList != null) { @@ -117,6 +119,7 @@ private static ClusterStats buildClusterStats( } clusterStatsBuilder.setTotalDroppedRequests(dropCount); } + clusterStatsBuilder.setLoadReportInterval(Durations.fromNanos(intervalNano)); return clusterStatsBuilder.build(); } @@ -159,74 +162,47 @@ private static void assertUpstreamLocalityStatsEqual(UpstreamLocalityStats expec } @Test - public void addAndGetAndRemoveLocality() { - loadStatsStore.addLocality(LOCALITY1); - assertThat(localityLoadCounters).containsKey(LOCALITY1); - - // Adding the same locality counter again causes an exception. - try { - loadStatsStore.addLocality(LOCALITY1); - Assert.fail(); - } catch (IllegalStateException expected) { - assertThat(expected).hasMessageThat() - .contains("An active counter for locality " + LOCALITY1 + " already exists"); - } - - assertThat(loadStatsStore.getLocalityCounter(LOCALITY1)) - .isSameInstanceAs(localityLoadCounters.get(LOCALITY1)); - assertThat(loadStatsStore.getLocalityCounter(LOCALITY2)).isNull(); - - // Removing an non-existing locality counter causes an exception. - try { - loadStatsStore.removeLocality(LOCALITY2); - Assert.fail(); - } catch (IllegalStateException expected) { - assertThat(expected).hasMessageThat() - .contains("No active counter for locality " + LOCALITY2 + " exists"); - } - - // Removing the locality counter only mark it as inactive, but not throw it away. - loadStatsStore.removeLocality(LOCALITY1); - assertThat(localityLoadCounters.get(LOCALITY1).isActive()).isFalse(); - - // Removing an inactive locality counter causes an exception. - try { - loadStatsStore.removeLocality(LOCALITY1); - Assert.fail(); - } catch (IllegalStateException expected) { - assertThat(expected).hasMessageThat() - .contains("No active counter for locality " + LOCALITY1 + " exists"); - } - - // Adding it back simply mark it as active again. + public void removeInactiveCountersAfterGeneratingLoadReport() { loadStatsStore.addLocality(LOCALITY1); - assertThat(localityLoadCounters.get(LOCALITY1).isActive()).isTrue(); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(1); + loadStatsStore.removeLocality(LOCALITY1); // becomes inactive + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(1); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(0); } @Test - public void removeInactiveCountersAfterGeneratingLoadReport() { - localityLoadCounters.put(LOCALITY1, new ClientLoadCounter()); - ClientLoadCounter inactiveCounter = new ClientLoadCounter(); - inactiveCounter.setActive(false); - localityLoadCounters.put(LOCALITY2, inactiveCounter); - loadStatsStore.generateLoadReport(); - assertThat(localityLoadCounters).containsKey(LOCALITY1); - assertThat(localityLoadCounters).doesNotContainKey(LOCALITY2); + public void localityCountersReferenceCounted() { + loadStatsStore.addLocality(LOCALITY1); + loadStatsStore.addLocality(LOCALITY1); + loadStatsStore.removeLocality(LOCALITY1); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(1); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()) + .isEqualTo(1); // still active + loadStatsStore.removeLocality(LOCALITY1); // becomes inactive + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(1); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(0); } @Test public void loadReportContainsRecordedStats() { - ClientLoadCounter counter1 = new ClientLoadCounter(4315, 3421, 23, 593); + ClientLoadCounter counter1 = loadStatsStore.addLocality(LOCALITY1); + counter1.setCallsSucceeded(4315); + counter1.setCallsInProgress(3421); + counter1.setCallsFailed(23); + counter1.setCallsIssued(593); counter1.recordMetric("cpu_utilization", 0.3244); counter1.recordMetric("mem_utilization", 0.01233); counter1.recordMetric("named_cost_or_utilization", 3221.6543); - ClientLoadCounter counter2 = new ClientLoadCounter(41234, 432, 431, 702); + ClientLoadCounter counter2 = loadStatsStore.addLocality(LOCALITY2); + counter2.setCallsSucceeded(41234); + counter2.setCallsInProgress(432); + counter2.setCallsFailed(431); + counter2.setCallsIssued(702); counter2.recordMetric("cpu_utilization", 0.6526); counter2.recordMetric("mem_utilization", 0.3473); counter2.recordMetric("named_cost_or_utilization", 87653.4234); - localityLoadCounters.put(LOCALITY1, counter1); - localityLoadCounters.put(LOCALITY2, counter2); + fakeClock.forwardNanos(1000L); Map metrics1 = ImmutableMap.of( "cpu_utilization", new MetricValue(1, 0.3244), @@ -245,16 +221,17 @@ public void loadReportContainsRecordedStats() { buildUpstreamLocalityStats(LOCALITY2, 41234, 432, 431, 702, buildEndpointLoadMetricStatsList(metrics2)) ), - null); + null, 1000L); assertClusterStatsEqual(expectedReport, loadStatsStore.generateLoadReport()); + fakeClock.forwardNanos(2000L); expectedReport = buildClusterStats( Arrays.asList( buildUpstreamLocalityStats(LOCALITY1, 0, 3421, 0, 0, null), buildUpstreamLocalityStats(LOCALITY2, 0, 432, 0, 0, null) ), - null); + null, 2000L); assertClusterStatsEqual(expectedReport, loadStatsStore.generateLoadReport()); } @@ -270,10 +247,13 @@ public void recordingDroppedRequests() { } assertThat(dropCounters.get("lb").get()).isEqualTo(numLbDrop); assertThat(dropCounters.get("throttle").get()).isEqualTo(numThrottleDrop); + + fakeClock.forwardNanos(1000L); ClusterStats expectedLoadReport = buildClusterStats(null, Arrays.asList(buildDroppedRequests("lb", numLbDrop), - buildDroppedRequests("throttle", numThrottleDrop))); + buildDroppedRequests("throttle", numThrottleDrop)), + 1000L); assertClusterStatsEqual(expectedLoadReport, loadStatsStore.generateLoadReport()); assertThat(dropCounters.get("lb").get()).isEqualTo(0); assertThat(dropCounters.get("throttle").get()).isEqualTo(0); diff --git a/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java b/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java index 4f9c8f51753..6c079b54edb 100644 --- a/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java +++ b/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java @@ -69,6 +69,7 @@ import io.grpc.xds.EnvoyProtoData.LbEndpoint; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.LocalityStore.LocalityStoreImpl; import io.grpc.xds.OrcaOobUtil.OrcaOobReportListener; import io.grpc.xds.OrcaOobUtil.OrcaReportingConfig; @@ -84,7 +85,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -1053,8 +1053,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { ArgumentCaptor subchannelPickerCaptor = ArgumentCaptor.forClass(null); inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) + .pickSubchannel(mock(PickSubchannelArgs.class)) + .getSubchannel()) .isSameInstanceAs(mockSubchannel3); // P0 gets READY - P0 R, P1 F&D, P2 R&D, P3 N/A @@ -1078,8 +1078,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { assertThat(deactivationTasks).hasSize(2); inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) + .pickSubchannel(mock(PickSubchannelArgs.class)) + .getSubchannel()) .isSameInstanceAs(mockSubchannel1); // P1 gets READY - P0 R, P1 R&D, P2 R&D, P3 N/A @@ -1188,8 +1188,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(2); inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) + .pickSubchannel(mock(PickSubchannelArgs.class)) + .getSubchannel()) .isSameInstanceAs(mockSubchannel22); // EDS update, localities moved: P0 sz1, sz3; P1 sz4; P2 sz2 - P0 C, P1 R, P2 R&D @@ -1214,8 +1214,8 @@ public void run() { assertThat(loadBalancers.values()).containsExactly(lb1, lb2, lb3, lb4); inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) + .pickSubchannel(mock(PickSubchannelArgs.class)) + .getSubchannel()) .isSameInstanceAs(mockSubchannel4); // The order of the following four handleResolvedAddresses() does not matter. We want to verify // they are after helper.updateBalancingState() @@ -1283,8 +1283,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(1); inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) + .pickSubchannel(mock(PickSubchannelArgs.class)) + .getSubchannel()) .isSameInstanceAs(newMockSubchannel3); inOrder.verifyNoMoreInteractions(); @@ -1337,9 +1337,11 @@ public ClusterStats generateLoadReport() { } @Override - public void addLocality(Locality locality) { + public ClientLoadCounter addLocality(Locality locality) { assertThat(localityCounters).doesNotContainKey(locality); - localityCounters.put(locality, new ClientLoadCounter()); + ClientLoadCounter counter = new ClientLoadCounter(); + localityCounters.put(locality, counter); + return counter; } @Override @@ -1348,12 +1350,6 @@ public void removeLocality(Locality locality) { localityCounters.remove(locality); } - @Nullable - @Override - public ClientLoadCounter getLocalityCounter(Locality locality) { - return localityCounters.get(locality); - } - @Override public void recordDroppedRequest(String category) { // NO-OP, verify by invocations. diff --git a/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java index e845a2ccc97..bea3a2f2faa 100644 --- a/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java @@ -40,6 +40,7 @@ import io.grpc.xds.ClientLoadCounter.LoadRecordingStreamTracerFactory; import io.grpc.xds.ClientLoadCounter.LoadRecordingSubchannelPicker; import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.LrsLoadBalancerProvider.LrsConfig; import java.net.SocketAddress; import java.util.ArrayDeque; @@ -72,8 +73,6 @@ public class LrsLoadBalancerTest { private static final String LRS_SERVER_NAME = "trafficdirector.googleapis.com"; private static final Locality TEST_LOCALITY = new Locality("test-region", "test-zone", "test-subzone"); - - private final ClientLoadCounter counter = new ClientLoadCounter(); private final LoadRecorder loadRecorder = new LoadRecorder(); private final Queue childBalancers = new ArrayDeque<>(); @@ -106,7 +105,7 @@ public void subchannelPickerInterceptedWithLoadRecording() { PickResult result = picker.pickSubchannel(mock(PickSubchannelArgs.class)); ClientStreamTracer.Factory tracerFactory = result.getStreamTracerFactory(); assertThat(((LoadRecordingStreamTracerFactory) tracerFactory).getCounter()) - .isSameInstanceAs(counter); + .isSameInstanceAs(loadRecorder.counter); loadBalancer.shutdown(); assertThat(childBalancer.shutdown).isTrue(); assertThat(loadRecorder.recording).isFalse(); @@ -300,7 +299,8 @@ public int hashCode() { } } - private final class LoadRecorder implements LoadStatsStore { + private static final class LoadRecorder implements LoadStatsStore { + private final ClientLoadCounter counter = new ClientLoadCounter(); private boolean recording = false; @Override @@ -309,9 +309,10 @@ public ClusterStats generateLoadReport() { } @Override - public void addLocality(Locality locality) { + public ClientLoadCounter addLocality(Locality locality) { assertThat(locality).isEqualTo(TEST_LOCALITY); recording = true; + return counter; } @Override @@ -320,12 +321,6 @@ public void removeLocality(Locality locality) { recording = false; } - @Override - public ClientLoadCounter getLocalityCounter(Locality locality) { - assertThat(locality).isEqualTo(TEST_LOCALITY); - return counter; - } - @Override public void recordDroppedRequest(String category) { throw new UnsupportedOperationException("should not be called"); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index 2a353c77f58..ec81022c924 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -39,7 +39,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; @@ -1866,7 +1865,8 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); assertThat(timeoutTask.isCancelled()).isTrue(); - verifyZeroInteractions(watcher3, watcher4); + // TODO(chengyuanzhang): migrate to verifyNoInteractions. + verifyNoMoreInteractions(watcher3, watcher4); } @Test @@ -2133,7 +2133,8 @@ public void multipleEndpointWatchers() { new LbEndpoint("192.168.0.1", 8080, 2, true)), 1, 0)); - verifyZeroInteractions(watcher3); + // TODO(chengyuanzhang): migrate to verifyNoInteractions. + verifyNoMoreInteractions(watcher3); // Management server sends back another EDS response contains ClusterLoadAssignment for the // other requested cluster. @@ -2517,7 +2518,8 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); assertThat(timeoutTask.isCancelled()).isTrue(); - verifyZeroInteractions(watcher3, watcher4); + // TODO(chengyuanzhang): migrate to verifyNoInteractions. + verifyNoMoreInteractions(watcher3, watcher4); } @Test @@ -3235,9 +3237,9 @@ public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { @Test public void reportLoadStatsToServer() { String clusterName = "cluster-foo.googleapis.com"; - LoadStatsStore loadStatsStore = new LoadStatsStoreImpl(clusterName, null); + xdsClient.addClientStats(clusterName, null); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(null); - xdsClient.reportClientStats(clusterName, null, loadStatsStore); + xdsClient.reportClientStats(); LoadReportCall lrsCall = loadReportCalls.poll(); verify(lrsCall.requestObserver).onNext(requestCaptor.capture()); assertThat(requestCaptor.getValue().getClusterStatsCount()) @@ -3253,12 +3255,14 @@ public void reportLoadStatsToServer() { ClusterStats report = Iterables.getOnlyElement(requestCaptor.getValue().getClusterStatsList()); assertThat(report.getClusterName()).isEqualTo(clusterName); - xdsClient.cancelClientStatsReport(clusterName, null); + xdsClient.removeClientStats(clusterName, null); fakeClock.forwardNanos(1000L); verify(lrsCall.requestObserver, times(3)).onNext(requestCaptor.capture()); assertThat(requestCaptor.getValue().getClusterStatsCount()) .isEqualTo(0); // no more stats reported + xdsClient.cancelClientStatsReport(); + assertThat(lrsEnded.get()).isTrue(); // See more test on LoadReportClientTest.java } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java index 178fe46bd42..fcd268e0193 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java @@ -39,7 +39,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; @@ -1875,7 +1874,8 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); assertThat(timeoutTask.isCancelled()).isTrue(); - verifyZeroInteractions(watcher3, watcher4); + // TODO(chengyuanzhang): migrate to verifyNoInteractions. + verifyNoMoreInteractions(watcher3, watcher4); } @Test @@ -2142,7 +2142,8 @@ public void multipleEndpointWatchers() { new LbEndpoint("192.168.0.1", 8080, 2, true)), 1, 0)); - verifyZeroInteractions(watcher3); + // TODO(chengyuanzhang): migrate to verifyNoInteractions. + verifyNoMoreInteractions(watcher3); // Management server sends back another EDS response contains ClusterLoadAssignment for the // other requested cluster. @@ -2526,7 +2527,8 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); assertThat(timeoutTask.isCancelled()).isTrue(); - verifyZeroInteractions(watcher3, watcher4); + // TODO(chengyuanzhang): migrate to verifyNoInteractions. + verifyNoMoreInteractions(watcher3, watcher4); } @Test @@ -3245,9 +3247,9 @@ public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { @Test public void reportLoadStatsToServer() { String clusterName = "cluster-foo.googleapis.com"; - LoadStatsStore loadStatsStore = new LoadStatsStoreImpl(clusterName, null); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(null); - xdsClient.reportClientStats(clusterName, null, loadStatsStore); + xdsClient.addClientStats(clusterName, null); + xdsClient.reportClientStats(); LoadReportCall lrsCall = loadReportCalls.poll(); verify(lrsCall.requestObserver).onNext(requestCaptor.capture()); assertThat(requestCaptor.getValue().getClusterStatsCount()) @@ -3263,12 +3265,14 @@ public void reportLoadStatsToServer() { ClusterStats report = Iterables.getOnlyElement(requestCaptor.getValue().getClusterStatsList()); assertThat(report.getClusterName()).isEqualTo(clusterName); - xdsClient.cancelClientStatsReport(clusterName, null); + xdsClient.removeClientStats(clusterName, null); fakeClock.forwardNanos(1000L); verify(lrsCall.requestObserver, times(3)).onNext(requestCaptor.capture()); assertThat(requestCaptor.getValue().getClusterStatsCount()) .isEqualTo(0); // no more stats reported + xdsClient.cancelClientStatsReport(); + assertThat(lrsEnded.get()).isTrue(); // See more test on LoadReportClientTest.java }