From cb07b0fb451e37100094f5c5ea7cf01e786af7fe Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Tue, 18 Aug 2020 12:30:05 -0700 Subject: [PATCH] xds: add data type for ClusterStats (#7335) In preparation of LRS v3 support. --- .../main/java/io/grpc/xds/EnvoyProtoData.java | 526 +++++++++++++++++- .../java/io/grpc/xds/LoadReportClient.java | 9 +- .../java/io/grpc/xds/LoadStatsManager.java | 4 +- .../java/io/grpc/xds/LoadStatsStoreImpl.java | 28 +- .../java/io/grpc/xds/EnvoyProtoDataTest.java | 79 +++ .../io/grpc/xds/LoadReportClientTest.java | 51 +- .../io/grpc/xds/LoadStatsStoreImplTest.java | 41 +- .../java/io/grpc/xds/LocalityStoreTest.java | 2 +- .../java/io/grpc/xds/LrsLoadBalancerTest.java | 2 +- 9 files changed, 664 insertions(+), 78 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java index cbb73423a16..e1872422da1 100644 --- a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java +++ b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java @@ -37,6 +37,7 @@ import io.grpc.xds.RouteMatch.PathMatcher; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -558,7 +559,7 @@ public String toString() { /** * See corresponding Envoy proto message {@link - * io.envoyproxy.envoy.api.v2.endpoint.LocalityLbEndpoints}. + * io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints}. */ static final class LocalityLbEndpoints { private final List endpoints; @@ -643,7 +644,8 @@ public String toString() { } /** - * See corresponding Envoy proto message {@link io.envoyproxy.envoy.api.v2.endpoint.LbEndpoint}. + * See corresponding Envoy proto message + * {@link io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint}. */ static final class LbEndpoint { private final EquivalentAddressGroup eag; @@ -735,7 +737,7 @@ public String toString() { /** * See corresponding Envoy proto message {@link - * io.envoyproxy.envoy.api.v2.ClusterLoadAssignment.Policy.DropOverload}. + * io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.Policy.DropOverload}. */ static final class DropOverload { private final String category; @@ -835,7 +837,7 @@ public String toString() { } } - /** See corresponding Envoy proto message {@link io.envoyproxy.envoy.api.v2.route.Route}. */ + /** See corresponding Envoy proto message {@link io.envoyproxy.envoy.config.route.v3.Route}. */ static final class Route { private final RouteMatch routeMatch; private final RouteAction routeAction; @@ -1068,7 +1070,9 @@ static StructOrError convertEnvoyProtoHeaderMatcher( } } - /** See corresponding Envoy proto message {@link io.envoyproxy.envoy.api.v2.route.RouteAction}. */ + /** + * See corresponding Envoy proto message {@link io.envoyproxy.envoy.config.route.v3.RouteAction}. + */ static final class RouteAction { private final long timeoutNano; // Exactly one of the following fields is non-null. @@ -1176,7 +1180,7 @@ static StructOrError fromEnvoyProtoRouteAction( /** * See corresponding Envoy proto message {@link - * io.envoyproxy.envoy.api.v2.route.WeightedCluster.ClusterWeight}. + * io.envoyproxy.envoy.config.route.v3.WeightedCluster.ClusterWeight}. */ static final class ClusterWeight { private final String name; @@ -1227,4 +1231,514 @@ static ClusterWeight fromEnvoyProtoClusterWeight( return new ClusterWeight(proto.getName(), proto.getWeight().getValue()); } } + + /** + * See corresponding Envoy proto message {@link + * io.envoyproxy.envoy.config.endpoint.v3.ClusterStats}. + */ + static final class ClusterStats { + private final String clusterName; + @Nullable + private final String clusterServiceName; + private final List upstreamLocalityStatsList; + private final List droppedRequestsList; + private final long totalDroppedRequests; + private final long loadReportIntervalNanos; + + private ClusterStats( + String clusterName, + @Nullable String clusterServiceName, + List upstreamLocalityStatsList, + List droppedRequestsList, + long totalDroppedRequests, + long loadReportIntervalNanos) { + this.clusterName = checkNotNull(clusterName, "clusterName"); + this.clusterServiceName = clusterServiceName; + this.upstreamLocalityStatsList = Collections.unmodifiableList( + checkNotNull(upstreamLocalityStatsList, "upstreamLocalityStatsList")); + this.droppedRequestsList = Collections.unmodifiableList( + checkNotNull(droppedRequestsList, "dropRequestsList")); + this.totalDroppedRequests = totalDroppedRequests; + this.loadReportIntervalNanos = loadReportIntervalNanos; + } + + String getClusterName() { + return clusterName; + } + + @Nullable + String getClusterServiceName() { + return clusterServiceName; + } + + List getUpstreamLocalityStatsList() { + return upstreamLocalityStatsList; + } + + List getDroppedRequestsList() { + return droppedRequestsList; + } + + long getTotalDroppedRequests() { + return totalDroppedRequests; + } + + long getLoadReportIntervalNanos() { + return loadReportIntervalNanos; + } + + io.envoyproxy.envoy.config.endpoint.v3.ClusterStats toEnvoyProtoClusterStats() { + io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.Builder builder = + io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.newBuilder(); + builder.setClusterName(clusterName); + if (clusterServiceName != null) { + builder.setClusterServiceName(clusterServiceName); + } + for (UpstreamLocalityStats upstreamLocalityStats : upstreamLocalityStatsList) { + builder.addUpstreamLocalityStats(upstreamLocalityStats.toEnvoyProtoUpstreamLocalityStats()); + } + for (DroppedRequests droppedRequests : droppedRequestsList) { + builder.addDroppedRequests(droppedRequests.toEnvoyProtoDroppedRequests()); + } + builder.setTotalDroppedRequests(totalDroppedRequests); + builder.setLoadReportInterval(Durations.fromNanos(loadReportIntervalNanos)); + return builder.build(); + } + + io.envoyproxy.envoy.api.v2.endpoint.ClusterStats toEnvoyProtoClusterStatsV2() { + io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.Builder builder = + io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.newBuilder(); + builder.setClusterName(clusterName); + for (UpstreamLocalityStats upstreamLocalityStats : upstreamLocalityStatsList) { + builder.addUpstreamLocalityStats( + upstreamLocalityStats.toEnvoyProtoUpstreamLocalityStatsV2()); + } + for (DroppedRequests droppedRequests : droppedRequestsList) { + builder.addDroppedRequests(droppedRequests.toEnvoyProtoDroppedRequestsV2()); + } + builder.setTotalDroppedRequests(totalDroppedRequests); + builder.setLoadReportInterval(Durations.fromNanos(loadReportIntervalNanos)); + return builder.build(); + } + + @VisibleForTesting + Builder toBuilder() { + Builder builder = new Builder() + .setClusterName(clusterName) + .setTotalDroppedRequests(totalDroppedRequests) + .setLoadReportIntervalNanos(loadReportIntervalNanos); + if (clusterServiceName != null) { + builder.setClusterServiceName(clusterServiceName); + } + for (UpstreamLocalityStats upstreamLocalityStats : upstreamLocalityStatsList) { + builder.addUpstreamLocalityStats(upstreamLocalityStats); + } + for (DroppedRequests droppedRequests : droppedRequestsList) { + builder.addDroppedRequests(droppedRequests); + } + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ClusterStats that = (ClusterStats) o; + return totalDroppedRequests == that.totalDroppedRequests + && loadReportIntervalNanos == that.loadReportIntervalNanos + && Objects.equals(clusterName, that.clusterName) + && Objects.equals(clusterServiceName, that.clusterServiceName) + && Objects.equals(upstreamLocalityStatsList, that.upstreamLocalityStatsList) + && Objects.equals(droppedRequestsList, that.droppedRequestsList); + } + + @Override + public int hashCode() { + return Objects.hash( + clusterName, clusterServiceName, upstreamLocalityStatsList, droppedRequestsList, + totalDroppedRequests, loadReportIntervalNanos); + } + + static Builder newBuilder() { + return new Builder(); + } + + static final class Builder { + private String clusterName; + private String clusterServiceName; + private final List upstreamLocalityStatsList = new ArrayList<>(); + private final List droppedRequestsList = new ArrayList<>(); + private long totalDroppedRequests; + private long loadReportIntervalNanos; + + private Builder() { + } + + Builder setClusterName(String clusterName) { + this.clusterName = checkNotNull(clusterName, "clusterName"); + return this; + } + + Builder setClusterServiceName(String clusterServiceName) { + this.clusterServiceName = checkNotNull(clusterServiceName, "clusterServiceName"); + return this; + } + + Builder setTotalDroppedRequests(long totalDroppedRequests) { + this.totalDroppedRequests = totalDroppedRequests; + return this; + } + + Builder setLoadReportIntervalNanos(long loadReportIntervalNanos) { + this.loadReportIntervalNanos = loadReportIntervalNanos; + return this; + } + + Builder addUpstreamLocalityStats(UpstreamLocalityStats upstreamLocalityStats) { + upstreamLocalityStatsList.add(checkNotNull(upstreamLocalityStats, "upstreamLocalityStats")); + return this; + } + + Builder addAllUpstreamLocalityStats(Collection upstreamLocalityStats) { + upstreamLocalityStatsList.addAll(upstreamLocalityStats); + return this; + } + + Builder addDroppedRequests(DroppedRequests droppedRequests) { + droppedRequestsList.add(checkNotNull(droppedRequests, "dropRequests")); + return this; + } + + ClusterStats build() { + return new ClusterStats( + clusterName, clusterServiceName,upstreamLocalityStatsList, droppedRequestsList, + totalDroppedRequests, loadReportIntervalNanos); + } + } + + /** + * See corresponding Envoy proto message {@link + * io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.DroppedRequests}. + */ + static final class DroppedRequests { + private final String category; + private final long droppedCount; + + DroppedRequests(String category, long droppedCount) { + this.category = checkNotNull(category, "category"); + this.droppedCount = droppedCount; + } + + String getCategory() { + return category; + } + + long getDroppedCount() { + return droppedCount; + } + + private io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.DroppedRequests + toEnvoyProtoDroppedRequests() { + return io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.DroppedRequests.newBuilder() + .setCategory(category) + .setDroppedCount(droppedCount) + .build(); + } + + private io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.DroppedRequests + toEnvoyProtoDroppedRequestsV2() { + return io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.DroppedRequests.newBuilder() + .setCategory(category) + .setDroppedCount(droppedCount) + .build(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DroppedRequests that = (DroppedRequests) o; + return droppedCount == that.droppedCount && Objects.equals(category, that.category); + } + + @Override + public int hashCode() { + return Objects.hash(category, droppedCount); + } + } + } + + /** + * See corresponding Envoy proto message {@link + * io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats}. + */ + static final class UpstreamLocalityStats { + private final Locality locality; + private final long totalSuccessfulRequests; + private final long totalErrorRequests; + private final long totalRequestsInProgress; + private final long totalIssuedRequests; + private final List loadMetricStatsList; + + private UpstreamLocalityStats( + Locality locality, + long totalSuccessfulRequests, + long totalErrorRequests, + long totalRequestsInProgress, + long totalIssuedRequests, + List loadMetricStatsList) { + this.locality = checkNotNull(locality, "locality"); + this.totalSuccessfulRequests = totalSuccessfulRequests; + this.totalErrorRequests = totalErrorRequests; + this.totalRequestsInProgress = totalRequestsInProgress; + this.totalIssuedRequests = totalIssuedRequests; + this.loadMetricStatsList = Collections.unmodifiableList( + checkNotNull(loadMetricStatsList, "loadMetricStatsList")); + } + + Locality getLocality() { + return locality; + } + + long getTotalSuccessfulRequests() { + return totalSuccessfulRequests; + } + + long getTotalErrorRequests() { + return totalErrorRequests; + } + + long getTotalRequestsInProgress() { + return totalRequestsInProgress; + } + + long getTotalIssuedRequests() { + return totalIssuedRequests; + } + + List getLoadMetricStatsList() { + return loadMetricStatsList; + } + + private io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats + toEnvoyProtoUpstreamLocalityStats() { + io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats.Builder builder + = io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats.newBuilder(); + builder + .setLocality(locality.toEnvoyProtoLocality()) + .setTotalSuccessfulRequests(totalSuccessfulRequests) + .setTotalErrorRequests(totalErrorRequests) + .setTotalRequestsInProgress(totalRequestsInProgress) + .setTotalIssuedRequests(totalIssuedRequests); + for (EndpointLoadMetricStats endpointLoadMetricStats : loadMetricStatsList) { + builder.addLoadMetricStats(endpointLoadMetricStats.toEnvoyProtoEndpointLoadMetricStats()); + } + return builder.build(); + } + + private io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats + toEnvoyProtoUpstreamLocalityStatsV2() { + io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats.Builder builder + = io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats.newBuilder(); + builder + .setLocality(locality.toEnvoyProtoLocalityV2()) + .setTotalSuccessfulRequests(totalSuccessfulRequests) + .setTotalErrorRequests(totalErrorRequests) + .setTotalRequestsInProgress(totalRequestsInProgress) + .setTotalIssuedRequests(totalIssuedRequests); + for (EndpointLoadMetricStats endpointLoadMetricStats : loadMetricStatsList) { + builder.addLoadMetricStats(endpointLoadMetricStats.toEnvoyProtoEndpointLoadMetricStatsV2()); + } + return builder.build(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + UpstreamLocalityStats that = (UpstreamLocalityStats) o; + return totalSuccessfulRequests == that.totalSuccessfulRequests + && totalErrorRequests == that.totalErrorRequests + && totalRequestsInProgress == that.totalRequestsInProgress + && totalIssuedRequests == that.totalIssuedRequests + && Objects.equals(locality, that.locality) + && Objects.equals(loadMetricStatsList, that.loadMetricStatsList); + } + + @Override + public int hashCode() { + return Objects.hash( + locality, totalSuccessfulRequests, totalErrorRequests, totalRequestsInProgress, + totalIssuedRequests, loadMetricStatsList); + } + + static Builder newBuilder() { + return new Builder(); + } + + static final class Builder { + private Locality locality; + private long totalSuccessfulRequests; + private long totalErrorRequests; + private long totalRequestsInProgress; + private long totalIssuedRequests; + private final List loadMetricStatsList = new ArrayList<>(); + + private Builder() { + } + + Builder setLocality(Locality locality) { + this.locality = checkNotNull(locality, "locality"); + return this; + } + + Builder setTotalSuccessfulRequests(long totalSuccessfulRequests) { + this.totalSuccessfulRequests = totalSuccessfulRequests; + return this; + } + + Builder setTotalErrorRequests(long totalErrorRequests) { + this.totalErrorRequests = totalErrorRequests; + return this; + } + + Builder setTotalRequestsInProgress(long totalRequestsInProgress) { + this.totalRequestsInProgress = totalRequestsInProgress; + return this; + } + + Builder setTotalIssuedRequests(long totalIssuedRequests) { + this.totalIssuedRequests = totalIssuedRequests; + return this; + } + + Builder addLoadMetricStats(EndpointLoadMetricStats endpointLoadMetricStats) { + loadMetricStatsList.add(checkNotNull(endpointLoadMetricStats, "endpointLoadMetricStats")); + return this; + } + + Builder addAllLoadMetricStats(Collection endpointLoadMetricStats) { + loadMetricStatsList.addAll( + checkNotNull(endpointLoadMetricStats, "endpointLoadMetricStats")); + return this; + } + + UpstreamLocalityStats build() { + return new UpstreamLocalityStats( + locality, totalSuccessfulRequests, totalErrorRequests, totalRequestsInProgress, + totalIssuedRequests, loadMetricStatsList); + } + } + } + + /** + * See corresponding Envoy proto message {@link + * io.envoyproxy.envoy.config.endpoint.v3.EndpointLoadMetricStats}. + */ + static final class EndpointLoadMetricStats { + private final String metricName; + private final long numRequestsFinishedWithMetric; + private final double totalMetricValue; + + private EndpointLoadMetricStats(String metricName, long numRequestsFinishedWithMetric, + double totalMetricValue) { + this.metricName = checkNotNull(metricName, "metricName"); + this.numRequestsFinishedWithMetric = numRequestsFinishedWithMetric; + this.totalMetricValue = totalMetricValue; + } + + String getMetricName() { + return metricName; + } + + long getNumRequestsFinishedWithMetric() { + return numRequestsFinishedWithMetric; + } + + double getTotalMetricValue() { + return totalMetricValue; + } + + private io.envoyproxy.envoy.config.endpoint.v3.EndpointLoadMetricStats + toEnvoyProtoEndpointLoadMetricStats() { + return io.envoyproxy.envoy.config.endpoint.v3.EndpointLoadMetricStats.newBuilder() + .setMetricName(metricName) + .setNumRequestsFinishedWithMetric(numRequestsFinishedWithMetric) + .setTotalMetricValue(totalMetricValue) + .build(); + } + + private io.envoyproxy.envoy.api.v2.endpoint.EndpointLoadMetricStats + toEnvoyProtoEndpointLoadMetricStatsV2() { + return io.envoyproxy.envoy.api.v2.endpoint.EndpointLoadMetricStats.newBuilder() + .setMetricName(metricName) + .setNumRequestsFinishedWithMetric(numRequestsFinishedWithMetric) + .setTotalMetricValue(totalMetricValue) + .build(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + EndpointLoadMetricStats that = (EndpointLoadMetricStats) o; + return numRequestsFinishedWithMetric == that.numRequestsFinishedWithMetric + && Double.compare(that.totalMetricValue, totalMetricValue) == 0 + && Objects.equals(metricName, that.metricName); + } + + @Override + public int hashCode() { + return Objects.hash(metricName, numRequestsFinishedWithMetric, totalMetricValue); + } + + static Builder newBuilder() { + return new Builder(); + } + + static final class Builder { + private String metricName; + private long numRequestsFinishedWithMetric; + private double totalMetricValue; + + private Builder() { + } + + Builder setMetricName(String metricName) { + this.metricName = checkNotNull(metricName, "metricName"); + return this; + } + + Builder setNumRequestsFinishedWithMetric(long numRequestsFinishedWithMetric) { + this.numRequestsFinishedWithMetric = numRequestsFinishedWithMetric; + return this; + } + + Builder setTotalMetricValue(double totalMetricValue) { + this.totalMetricValue = totalMetricValue; + return this; + } + + EndpointLoadMetricStats build() { + return new EndpointLoadMetricStats( + metricName, numRequestsFinishedWithMetric, totalMetricValue); + } + } + } } diff --git a/xds/src/main/java/io/grpc/xds/LoadReportClient.java b/xds/src/main/java/io/grpc/xds/LoadReportClient.java index 84d2b340cbd..da526147902 100644 --- a/xds/src/main/java/io/grpc/xds/LoadReportClient.java +++ b/xds/src/main/java/io/grpc/xds/LoadReportClient.java @@ -37,6 +37,7 @@ import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.BackoffPolicy; import io.grpc.stub.StreamObserver; +import io.grpc.xds.EnvoyProtoData.ClusterStats; import io.grpc.xds.XdsLogger.XdsLogLevel; import java.util.List; import java.util.concurrent.ScheduledExecutorService; @@ -228,10 +229,14 @@ public void run() { private void sendLoadReport() { LoadStatsRequest.Builder requestBuilder = LoadStatsRequest.newBuilder().setNode(node); if (reportAllClusters) { - requestBuilder.addAllClusterStats(loadStatsManager.getAllLoadReports()); + for (ClusterStats clusterStats : loadStatsManager.getAllLoadReports()) { + requestBuilder.addClusterStats(clusterStats.toEnvoyProtoClusterStatsV2()); + } } else { for (String name : clusterNames) { - requestBuilder.addAllClusterStats(loadStatsManager.getClusterLoadReports(name)); + for (ClusterStats clusterStats : loadStatsManager.getClusterLoadReports(name)) { + requestBuilder.addClusterStats(clusterStats.toEnvoyProtoClusterStatsV2()); + } } } LoadStatsRequest request = requestBuilder.build(); diff --git a/xds/src/main/java/io/grpc/xds/LoadStatsManager.java b/xds/src/main/java/io/grpc/xds/LoadStatsManager.java index 5372d37b0ea..02fd466d3a6 100644 --- a/xds/src/main/java/io/grpc/xds/LoadStatsManager.java +++ b/xds/src/main/java/io/grpc/xds/LoadStatsManager.java @@ -19,7 +19,7 @@ import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; +import io.grpc.xds.EnvoyProtoData.ClusterStats; import io.grpc.xds.EnvoyProtoData.Locality; import java.util.ArrayList; import java.util.HashMap; @@ -90,7 +90,6 @@ void removeLoadStats(String cluster, @Nullable String clusterService) { * the interval between calls of this method or {@link #getAllLoadReports}. A cluster may send * loads to more than one cluster_service, they are included in separate stats reports. */ - // TODO(chengyuanzhang): do not use proto type directly. List getClusterLoadReports(String cluster) { List res = new ArrayList<>(); Map> clusterLoadStatsStores = @@ -109,7 +108,6 @@ List getClusterLoadReports(String cluster) { * interval between calls of this method or {@link #getClusterLoadReports}. Each report * includes stats for one cluster:cluster_service. */ - // TODO(chengyuanzhang): do not use proto type directly. List getAllLoadReports() { List res = new ArrayList<>(); for (Map> clusterLoadStatsStores diff --git a/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java b/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java index 6a8b5447216..afb91c3ecc7 100644 --- a/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java +++ b/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java @@ -20,15 +20,14 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; -import com.google.protobuf.util.Durations; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.DroppedRequests; -import io.envoyproxy.envoy.api.v2.endpoint.EndpointLoadMetricStats; -import io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats; import io.grpc.internal.GrpcUtil; import io.grpc.xds.ClientLoadCounter.ClientLoadSnapshot; import io.grpc.xds.ClientLoadCounter.MetricValue; +import io.grpc.xds.EnvoyProtoData.ClusterStats; +import io.grpc.xds.EnvoyProtoData.ClusterStats.DroppedRequests; +import io.grpc.xds.EnvoyProtoData.EndpointLoadMetricStats; import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.UpstreamLocalityStats; import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.LoadStatsManager.LoadStatsStoreFactory; import java.util.Map; @@ -50,7 +49,6 @@ final class LoadStatsStoreImpl implements LoadStatsStore { private final String clusterName; @Nullable - @SuppressWarnings("unused") private final String clusterServiceName; private final ConcurrentMap> localityLoadCounters = new ConcurrentHashMap<>(); @@ -80,12 +78,14 @@ final class LoadStatsStoreImpl implements LoadStatsStore { public ClusterStats generateLoadReport() { ClusterStats.Builder statsBuilder = ClusterStats.newBuilder(); statsBuilder.setClusterName(clusterName); - // TODO(chengyuangzhang): also set cluster_service_name if provided. + if (clusterServiceName != null) { + statsBuilder.setClusterServiceName(clusterServiceName); + } for (Map.Entry> entry : localityLoadCounters.entrySet()) { ClientLoadSnapshot snapshot = entry.getValue().get().snapshot(); UpstreamLocalityStats.Builder localityStatsBuilder = - UpstreamLocalityStats.newBuilder().setLocality(entry.getKey().toEnvoyProtoLocalityV2()); + UpstreamLocalityStats.newBuilder().setLocality(entry.getKey()); localityStatsBuilder .setTotalSuccessfulRequests(snapshot.getCallsSucceeded()) .setTotalErrorRequests(snapshot.getCallsFailed()) @@ -96,9 +96,10 @@ public ClusterStats generateLoadReport() { EndpointLoadMetricStats.newBuilder() .setMetricName(metric.getKey()) .setNumRequestsFinishedWithMetric(metric.getValue().getNumReports()) - .setTotalMetricValue(metric.getValue().getTotalValue())); + .setTotalMetricValue(metric.getValue().getTotalValue()) + .build()); } - statsBuilder.addUpstreamLocalityStats(localityStatsBuilder); + statsBuilder.addUpstreamLocalityStats(localityStatsBuilder.build()); // Discard counters for localities that are no longer exposed by the remote balancer and // no RPCs ongoing. if (entry.getValue().getReferenceCount() == 0 && snapshot.getCallsInProgress() == 0) { @@ -109,13 +110,10 @@ public ClusterStats generateLoadReport() { for (Map.Entry entry : dropCounters.entrySet()) { long drops = entry.getValue().getAndSet(0); totalDrops += drops; - statsBuilder.addDroppedRequests(DroppedRequests.newBuilder() - .setCategory(entry.getKey()) - .setDroppedCount(drops)); + statsBuilder.addDroppedRequests(new DroppedRequests(entry.getKey(),drops)); } statsBuilder.setTotalDroppedRequests(totalDrops); - statsBuilder.setLoadReportInterval( - Durations.fromNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS))); + statsBuilder.setLoadReportIntervalNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS)); stopwatch.reset().start(); return statsBuilder.build(); } diff --git a/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java b/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java index 24e739bf560..2e307d6c7eb 100644 --- a/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java +++ b/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java @@ -34,12 +34,16 @@ import io.envoyproxy.envoy.type.v3.FractionalPercent; import io.envoyproxy.envoy.type.v3.Int64Range; import io.grpc.xds.EnvoyProtoData.Address; +import io.grpc.xds.EnvoyProtoData.ClusterStats; +import io.grpc.xds.EnvoyProtoData.ClusterStats.DroppedRequests; import io.grpc.xds.EnvoyProtoData.ClusterWeight; +import io.grpc.xds.EnvoyProtoData.EndpointLoadMetricStats; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.Route; import io.grpc.xds.EnvoyProtoData.RouteAction; import io.grpc.xds.EnvoyProtoData.StructOrError; +import io.grpc.xds.EnvoyProtoData.UpstreamLocalityStats; import io.grpc.xds.RouteMatch.FractionMatcher; import io.grpc.xds.RouteMatch.HeaderMatcher; import io.grpc.xds.RouteMatch.PathMatcher; @@ -564,4 +568,79 @@ public void convertClusterWeight() { assertThat(struct.getName()).isEqualTo("cluster-foo"); assertThat(struct.getWeight()).isEqualTo(30); } + + @Test + public void clusterStats_convertToEnvoyProto() { + ClusterStats clusterStats = + ClusterStats.newBuilder() + .setClusterName("cluster1") + .setLoadReportIntervalNanos(1234) + .setTotalDroppedRequests(123) + .addUpstreamLocalityStats(UpstreamLocalityStats.newBuilder() + .setLocality(new Locality("region1", "zone1", "subzone1")) + .setTotalErrorRequests(1) + .setTotalRequestsInProgress(2) + .setTotalSuccessfulRequests(100) + .setTotalIssuedRequests(103) + .addLoadMetricStats(EndpointLoadMetricStats.newBuilder() + .setMetricName("metric1") + .setNumRequestsFinishedWithMetric(1000) + .setTotalMetricValue(0.5D) + .build()) + .build()) + .addDroppedRequests(new DroppedRequests("category1", 100)) + .build(); + + io.envoyproxy.envoy.config.endpoint.v3.ClusterStats clusterStatsProto = + clusterStats.toEnvoyProtoClusterStats(); + assertThat(clusterStatsProto).isEqualTo( + io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.newBuilder() + .setClusterName("cluster1") + .setLoadReportInterval(Durations.fromNanos(1234)) + .setTotalDroppedRequests(123) + .addUpstreamLocalityStats( + io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats.newBuilder() + .setLocality( + new Locality("region1", "zone1", "subzone1").toEnvoyProtoLocality()) + .setTotalErrorRequests(1) + .setTotalRequestsInProgress(2) + .setTotalSuccessfulRequests(100) + .setTotalIssuedRequests(103) + .addLoadMetricStats( + io.envoyproxy.envoy.config.endpoint.v3.EndpointLoadMetricStats.newBuilder() + .setMetricName("metric1") + .setNumRequestsFinishedWithMetric(1000) + .setTotalMetricValue(0.5D))) + .addDroppedRequests( + io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.DroppedRequests.newBuilder() + .setCategory("category1") + .setDroppedCount(100)) + .build()); + + io.envoyproxy.envoy.api.v2.endpoint.ClusterStats clusterStatsProtoV2 = + clusterStats.toEnvoyProtoClusterStatsV2(); + assertThat(clusterStatsProtoV2).isEqualTo( + io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.newBuilder() + .setClusterName("cluster1") + .setLoadReportInterval(Durations.fromNanos(1234)) + .setTotalDroppedRequests(123) + .addUpstreamLocalityStats( + io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats.newBuilder() + .setLocality( + new Locality("region1", "zone1", "subzone1").toEnvoyProtoLocalityV2()) + .setTotalErrorRequests(1) + .setTotalRequestsInProgress(2) + .setTotalSuccessfulRequests(100) + .setTotalIssuedRequests(103) + .addLoadMetricStats( + io.envoyproxy.envoy.api.v2.endpoint.EndpointLoadMetricStats.newBuilder() + .setMetricName("metric1") + .setNumRequestsFinishedWithMetric(1000) + .setTotalMetricValue(0.5D))) + .addDroppedRequests( + io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.DroppedRequests.newBuilder() + .setCategory("category1") + .setDroppedCount(100)) + .build()); + } } diff --git a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java index be68f08b110..a63171ef447 100644 --- a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java +++ b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java @@ -35,11 +35,7 @@ import com.google.protobuf.Struct; import com.google.protobuf.Value; import com.google.protobuf.util.Durations; -import io.envoyproxy.envoy.api.v2.core.Locality; import io.envoyproxy.envoy.api.v2.core.Node; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.DroppedRequests; -import io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats; import io.envoyproxy.envoy.service.load_stats.v2.LoadReportingServiceGrpc; import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest; import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsResponse; @@ -54,6 +50,10 @@ import io.grpc.internal.FakeClock; import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; +import io.grpc.xds.EnvoyProtoData.ClusterStats; +import io.grpc.xds.EnvoyProtoData.ClusterStats.DroppedRequests; +import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.UpstreamLocalityStats; import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.LoadStatsManager.LoadStatsStoreFactory; import java.util.ArrayDeque; @@ -225,7 +225,7 @@ public void periodicLoadReporting() { fakeClock.forwardNanos(1); assertThat(loadStatsStore1.reported).hasSize(1); ClusterStats report1 = loadStatsStore1.reported.poll(); - assertThat(Durations.toNanos(report1.getLoadReportInterval())).isEqualTo(1000); + assertThat(report1.getLoadReportIntervalNanos()).isEqualTo(1000); inOrder.verify(requestObserver) .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report1)))); @@ -233,7 +233,7 @@ public void periodicLoadReporting() { fakeClock.forwardNanos(1000); assertThat(loadStatsStore1.reported).hasSize(1); report1 = loadStatsStore1.reported.poll(); - assertThat(Durations.toNanos(report1.getLoadReportInterval())).isEqualTo(1000); + assertThat(report1.getLoadReportIntervalNanos()).isEqualTo(1000); inOrder.verify(requestObserver) .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report1)))); @@ -250,7 +250,7 @@ public void periodicLoadReporting() { fakeClock.forwardNanos(1000); assertThat(loadStatsStore1.reported).hasSize(1); report1 = loadStatsStore1.reported.poll(); - assertThat(Durations.toNanos(report1.getLoadReportInterval())).isEqualTo(2000); + assertThat(report1.getLoadReportIntervalNanos()).isEqualTo(2000); assertThat(loadStatsStore2.reported).isEmpty(); inOrder.verify(requestObserver) .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report1)))); @@ -269,8 +269,8 @@ public void periodicLoadReporting() { report1 = loadStatsStore1.reported.poll(); assertThat(loadStatsStore2.reported).hasSize(1); ClusterStats report2 = loadStatsStore2.reported.poll(); - assertThat(Durations.toNanos(report1.getLoadReportInterval())).isEqualTo(2000); - assertThat(Durations.toNanos(report2.getLoadReportInterval())).isEqualTo(2000 + 2000); + assertThat(report1.getLoadReportIntervalNanos()).isEqualTo(2000); + assertThat(report2.getLoadReportIntervalNanos()).isEqualTo(2000 + 2000); inOrder.verify(requestObserver) .onNext(argThat(new LoadStatsRequestMatcher(Arrays.asList(report1, report2)))); @@ -283,7 +283,7 @@ public void periodicLoadReporting() { assertThat(loadStatsStore1.reported).isEmpty(); assertThat(loadStatsStore2.reported).hasSize(1); report2 = loadStatsStore2.reported.poll(); - assertThat(Durations.toNanos(report2.getLoadReportInterval())).isEqualTo(2000); + assertThat(report2.getLoadReportIntervalNanos()).isEqualTo(2000); inOrder.verify(requestObserver) .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report2)))); @@ -399,7 +399,7 @@ public void lrsStreamClosedAndRetried() { .onNext(buildLrsResponse(ImmutableList.of(clusterName), 10)); fakeClock.forwardNanos(10); ClusterStats report = Iterables.getOnlyElement(loadStatsStore.reported); - assertThat(Durations.toNanos(report.getLoadReportInterval())) + assertThat(report.getLoadReportIntervalNanos()) .isEqualTo(TimeUnit.SECONDS.toNanos(1 + 10 + 2) + 10); verify(requestObserver) .onNext(argThat(new LoadStatsRequestMatcher(Collections.singletonList(report)))); @@ -500,8 +500,9 @@ public boolean matches(LoadStatsRequest argument) { if (argument.getClusterStatsCount() != expectedStats.size()) { return false; } - for (ClusterStats stats : argument.getClusterStatsList()) { - if (!stats.equals(expectedStats.get(stats.getClusterName()))) { + for (io.envoyproxy.envoy.api.v2.endpoint.ClusterStats stats + : argument.getClusterStatsList()) { + if (!stats.equals(expectedStats.get(stats.getClusterName()).toEnvoyProtoClusterStatsV2())) { return false; } } @@ -528,7 +529,7 @@ private FakeLoadStatsStore(String cluster, String clusterService, Stopwatch stop public ClusterStats generateLoadReport() { ClusterStats report = stats.toBuilder() - .setLoadReportInterval(Durations.fromNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS))) + .setLoadReportIntervalNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS)) .build(); stopwatch.reset().start(); reported.offer(report); @@ -563,25 +564,19 @@ private void refresh() { if (clusterService != null) { clusterStatsBuilder.setClusterServiceName(clusterService); } - clusterStatsBuilder.addUpstreamLocalityStats( - UpstreamLocalityStats.newBuilder() - .setLocality( - Locality.newBuilder() - .setRegion(cluster + "-region-foo") - .setZone(cluster + "-zone-bar") - .setSubZone(cluster + "-subzone-baz")) + clusterStatsBuilder + .addUpstreamLocalityStats(UpstreamLocalityStats.newBuilder() + .setLocality(new Locality( + cluster + "-region-foo", cluster + "-zone-bar", cluster + "-subzone-baz")) .setTotalRequestsInProgress(callsInProgress) .setTotalSuccessfulRequests(callsSucceeded) .setTotalErrorRequests(callsFailed) - .setTotalIssuedRequests(callsIssued)) + .setTotalIssuedRequests(callsIssued) + .build()) .addDroppedRequests( - DroppedRequests.newBuilder() - .setCategory("lb") - .setDroppedCount(numLbDrops)) + new DroppedRequests("lb",numLbDrops)) .addDroppedRequests( - DroppedRequests.newBuilder() - .setCategory("throttle") - .setDroppedCount(numThrottleDrops)) + new DroppedRequests("throttle", numThrottleDrops)) .setTotalDroppedRequests(numLbDrops + numThrottleDrops); stats = clusterStatsBuilder.build(); } diff --git a/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java b/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java index 4320ff279fd..127124afc2f 100644 --- a/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java +++ b/xds/src/test/java/io/grpc/xds/LoadStatsStoreImplTest.java @@ -20,14 +20,13 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; -import com.google.protobuf.util.Durations; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.DroppedRequests; -import io.envoyproxy.envoy.api.v2.endpoint.EndpointLoadMetricStats; -import io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats; import io.grpc.internal.FakeClock; import io.grpc.xds.ClientLoadCounter.MetricValue; +import io.grpc.xds.EnvoyProtoData.ClusterStats; +import io.grpc.xds.EnvoyProtoData.ClusterStats.DroppedRequests; +import io.grpc.xds.EnvoyProtoData.EndpointLoadMetricStats; import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.UpstreamLocalityStats; import io.grpc.xds.LoadStatsManager.LoadStatsStore; import java.util.ArrayList; import java.util.Arrays; @@ -85,7 +84,7 @@ private static UpstreamLocalityStats buildUpstreamLocalityStats( @Nullable List metrics) { UpstreamLocalityStats.Builder builder = UpstreamLocalityStats.newBuilder() - .setLocality(locality.toEnvoyProtoLocalityV2()) + .setLocality(locality) .setTotalSuccessfulRequests(callsSucceed) .setTotalErrorRequests(callsFailed) .setTotalRequestsInProgress(callsInProgress) @@ -97,10 +96,7 @@ private static UpstreamLocalityStats buildUpstreamLocalityStats( } private static DroppedRequests buildDroppedRequests(String category, long counts) { - return DroppedRequests.newBuilder() - .setCategory(category) - .setDroppedCount(counts) - .build(); + return new DroppedRequests(category, counts); } private static ClusterStats buildClusterStats( @@ -119,15 +115,16 @@ private static ClusterStats buildClusterStats( } clusterStatsBuilder.setTotalDroppedRequests(dropCount); } - clusterStatsBuilder.setLoadReportInterval(Durations.fromNanos(intervalNano)); + clusterStatsBuilder.setLoadReportIntervalNanos(intervalNano); return clusterStatsBuilder.build(); } private static void assertClusterStatsEqual(ClusterStats expected, ClusterStats actual) { assertThat(actual.getClusterName()).isEqualTo(expected.getClusterName()); - assertThat(actual.getLoadReportInterval()).isEqualTo(expected.getLoadReportInterval()); + assertThat(actual.getLoadReportIntervalNanos()) + .isEqualTo(expected.getLoadReportIntervalNanos()); assertThat(actual.getTotalDroppedRequests()).isEqualTo(expected.getTotalDroppedRequests()); - assertThat(actual.getDroppedRequestsCount()).isEqualTo(expected.getDroppedRequestsCount()); + assertThat(actual.getDroppedRequestsList()).hasSize(expected.getDroppedRequestsList().size()); assertThat(new HashSet<>(actual.getDroppedRequestsList())) .isEqualTo(new HashSet<>(expected.getDroppedRequestsList())); assertUpstreamLocalityStatsListsEqual(actual.getUpstreamLocalityStatsList(), @@ -137,7 +134,7 @@ private static void assertClusterStatsEqual(ClusterStats expected, ClusterStats private static void assertUpstreamLocalityStatsListsEqual(List expected, List actual) { assertThat(actual).hasSize(expected.size()); - Map expectedLocalityStats = + Map expectedLocalityStats = new HashMap<>(); for (UpstreamLocalityStats stats : expected) { expectedLocalityStats.put(stats.getLocality(), stats); @@ -164,10 +161,10 @@ private static void assertUpstreamLocalityStatsEqual(UpstreamLocalityStats expec @Test public void removeInactiveCountersAfterGeneratingLoadReport() { loadStatsStore.addLocality(LOCALITY1); - assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(1); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsList()).hasSize(1); loadStatsStore.removeLocality(LOCALITY1); // becomes inactive - assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(1); - assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(0); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsList()).hasSize(1); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsList()).isEmpty(); } @Test @@ -175,12 +172,12 @@ public void localityCountersReferenceCounted() { loadStatsStore.addLocality(LOCALITY1); loadStatsStore.addLocality(LOCALITY1); loadStatsStore.removeLocality(LOCALITY1); - assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(1); - assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()) - .isEqualTo(1); // still active + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsList()).hasSize(1); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsList()) + .hasSize(1); // still active loadStatsStore.removeLocality(LOCALITY1); // becomes inactive - assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(1); - assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsCount()).isEqualTo(0); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsList()).hasSize(1); + assertThat(loadStatsStore.generateLoadReport().getUpstreamLocalityStatsList()).isEmpty(); } @Test diff --git a/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java b/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java index 6c079b54edb..f514cf1f1d6 100644 --- a/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java +++ b/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java @@ -42,7 +42,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; import io.grpc.ChannelLogger; import io.grpc.ClientStreamTracer; import io.grpc.ConnectivityState; @@ -65,6 +64,7 @@ import io.grpc.internal.FakeClock.TaskFilter; import io.grpc.xds.ClientLoadCounter.LoadRecordingStreamTracerFactory; import io.grpc.xds.ClientLoadCounter.MetricsRecordingListener; +import io.grpc.xds.EnvoyProtoData.ClusterStats; import io.grpc.xds.EnvoyProtoData.DropOverload; import io.grpc.xds.EnvoyProtoData.LbEndpoint; import io.grpc.xds.EnvoyProtoData.Locality; diff --git a/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java index bea3a2f2faa..e53ca3048c7 100644 --- a/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; import io.grpc.Attributes; import io.grpc.ClientStreamTracer; import io.grpc.ConnectivityState; @@ -39,6 +38,7 @@ import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.xds.ClientLoadCounter.LoadRecordingStreamTracerFactory; import io.grpc.xds.ClientLoadCounter.LoadRecordingSubchannelPicker; +import io.grpc.xds.EnvoyProtoData.ClusterStats; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.LrsLoadBalancerProvider.LrsConfig;